Merge pull request #1737 from grycap/devel

Devel

Author: micafer

IM/CloudInfo.py

@@ -98,7 +98,7 @@ def get_cloud_list(auth_data):
         res = []
 
         for i, auth in enumerate(auth_data.auth_list):
-            if 'type' in auth and auth['type'] not in ['InfrastructureManager', 'VMRC', 'AppDBIS']:
+            if 'type' in auth and auth['type'] not in ['InfrastructureManager', 'VMRC', 'AppDBIS', 'EGIIS']:
                 cloud_item = CloudInfo()
                 cloud_item.type = auth['type']
                 if 'id' in auth.keys() and auth['id']:

IM/FedcloudInfo.py

@@ -187,26 +187,11 @@ def list_images(filters=None, do_order=True):
         if not filters:
             filters = {}
 
-        if "app" in filters:
-            app = filters["app"]
-        else:
-            app = ""
-
-        if "distribution" in filters:
-            distribution = filters["distribution"]
-        else:
-            distribution = ".*"
-
         if "vo" in filters:
             vo = filters["vo"]
         else:
             vo = None
 
-        if "version" in filters:
-            version = filters["version"]
-        else:
-            version = ".*"
-
         params = {}
         if vo:
             params.update({"vo_name": vo})
@@ -216,22 +201,23 @@ def list_images(filters=None, do_order=True):
 
         res = []
         for image in images:
-            app_name_reg = r".*%s.* \[%s\/%s\/.*]" % (
-                app.lower(),
-                distribution.lower(),
-                version,
-            )
-            if not re.search(app_name_reg, image["name"].lower()):
-                continue
-
-            endpoint = urlparse(image["endpoint"])
-            res.append(
-                {
-                    "uri": "ost://%s/%s" % (endpoint[1], image["id"]),
-                    "name": image["name"],
-                    "vo": image["vo"],
-                }
-            )
+            image_name = image["name"].lower()
+            accept = True
+            for elem in ["distribution", "version", "app"]:
+                if filters.get(elem) and filters.get(elem).lower() not in image_name:
+                    accept = False
+                    break
+
+            if accept:
+                endpoint = urlparse(image["endpoint"])
+                res.append(
+                    {
+                        "uri": "ost://%s/%s" % (endpoint[1], image["id"]),
+                        "name": image["name"],
+                        "vo": image["vo"],
+                    }
+                )
+
         if vo and do_order:
             sites = FedcloudInfo.get_sites_supporting_vo(vo)
             if sites:

IM/InfrastructureManager.py

@@ -417,7 +417,7 @@ def search_vm(inf, radl_sys, auth):
     def systems_with_iis(sel_inf, radl, auth):
         """
         Concrete systems using Image Information Systems.
-        Currently supported VMRC and AppDBIS
+        Currently supported VMRC and AppDBIS/EGIIS
         NOTE: consider not-fake deploys (vm_number > 0)
         """
         # Get VMRC credentials
@@ -427,9 +427,12 @@ def systems_with_iis(sel_inf, radl, auth):
                 vmrc_list.append(VMRC(vmrc_elem['host'], vmrc_elem['username'], vmrc_elem['password']))
 
         # Get AppDBIS credentials
-        appdbis_list = []
-        for appdbis_elem in auth.getAuthInfo('AppDBIS'):
-            appdbis_list.append(FedcloudInfo)
+        egiis_list = []
+        for _ in auth.getAuthInfo('AppDBIS'):
+            egiis_list.append(FedcloudInfo)
+        # Get EGIIS credentials
+        for _ in auth.getAuthInfo('EGIIS'):
+            egiis_list.append(FedcloudInfo)
 
         systems_with_vmrc = {}
         for system_id in set([d.id for d in radl.deploys if d.vm_number > 0]):
@@ -455,22 +458,22 @@ def systems_with_iis(sel_inf, radl, auth):
                     s_without_apps.addFeature(f)
 
             vmrc_res = [s0 for vmrc in vmrc_list for s0 in vmrc.search_vm(s)]
-            appdbis_res = [s0 for appdbis in appdbis_list for s0 in appdbis.search_vm(s)]
+            egiis_res = [s0 for egiis in egiis_list for s0 in egiis.search_vm(s)]
             local_msg, local_res = InfrastructureManager.search_vm(sel_inf, s, auth)
             # Check that now the image URL is in the RADL
-            if not s.getValue("disk.0.image.url") and not vmrc_res and not appdbis_res and not local_res:
+            if not s.getValue("disk.0.image.url") and not vmrc_res and not egiis_res and not local_res:
                 msg = "No VMI obtained from Sites"
                 if vmrc_list:
                     msg += " nor VMRC"
-                if appdbis_list:
-                    msg += " nor AppDBIS"
+                if egiis_list:
+                    msg += " nor EGI IS"
                 msg += " to system '" + system_id + "'"
                 if local_msg:
                     msg += ": " + local_msg
                 raise Exception(msg)
 
             n = [s_without_apps.clone().applyFeatures(s0, conflict="other", missing="other")
-                 for s0 in (vmrc_res + appdbis_res + local_res)]
+                 for s0 in (vmrc_res + egiis_res + local_res)]
             systems_with_vmrc[system_id] = n if n else [s_without_apps]
 
         return systems_with_vmrc

IM/connectors/CloudConnector.py

@@ -819,3 +819,13 @@ def convert_memory_unit(memory, unit="M"):
         if converted - int(converted) < 0.0000000000001:
             converted = int(converted)
         return converted
+
+    @staticmethod
+    def loose_version_compare(user_version, system_version):
+        pattern = r'^\d+(?:\.\d+){0,2}$'
+        if re.match(pattern, user_version) and re.match(pattern, system_version):
+            u = [int(x) for x in user_version.split(".")]
+            s = [int(x) for x in system_version.split(".")]
+            return s[:len(u)] == u
+        else:
+            return user_version == system_version

IM/connectors/OpenStack.py

@@ -2164,7 +2164,7 @@ def _filter_images(self, image_list, filters=None):
             if version is not None:
                 image_version = image.extra.get('os_version', None)
                 if image_version:
-                    if version.lower() != image_version.lower():
+                    if not self.loose_version_compare(version, image_version):
                         continue
                 elif version.lower() not in image.name.lower():
                     continue

doc/source/dashboard.rst

@@ -26,7 +26,7 @@ How to deploy the dashboard
 Register a client in an OIDC server with the following properties:
 
 - redirect uri: `https://<DASHBOARD_HOST>:<PORT>/login/oidc/authorized`
-- scopes: 'openid', 'email', 'profile', 'offline_access' ('eduperson_entitlement' in EGI Check-In optional)
+- scopes: 'openid', 'email', 'profile', 'offline_access' ('entitlements' in EGI Check-In optional)
 - introspection endpoint enabled
 
 Create the `config.json` file (see the [example](app/config-sample.json)) setting the following variables:
@@ -441,4 +441,4 @@ show in :ref:`Fig. 15 <figure_dash_vm_resize>`.
 .. _figure_dash_vm_resize:
 .. figure:: images/dash_vm_resize.png
 
-   Fig 16. VM Resize modal form.
+   Fig 16. VM Resize modal form.

doc/source/gstarted.rst

@@ -62,7 +62,7 @@ Cloud providers:
    * **aws://<region>/<ami-id>**, for Amazon Web Service;
    * **gce://<region>/<image-id>**, for Google Cloud;
    * **azr://<publisher>/<offer>/<sku>/<version>**, for Microsoft Azure; and
-   * **appdb://<site_name>/<apc_name>?<vo_name>**, for EGI or OpenStack connector using AppDB info.
+   * **egi://<site_name>/<apc_name>?<vo_name>**, for EGI or OpenStack connector using AppDB info.
    * **docker://<docker_image>**, for Docker images.
    * **fbw://<fns_server>/<image-id>**, for FogBow images.
    * **lin://linode/<image-id>**, for Linode images.

doc/source/radl.rst

@@ -328,13 +328,14 @@ machine.  The supported features are:
    * ``aws://<region>/<snapshot-id>``, for Amazon Web Service;
    * ``aws://<region>/<snapshot-name>``, for Amazon Web Service;
    * ``gce://<region>/<image-id>``, for Google Cloud;
-   * ``azr://<image-id>``, for Microsoft Azure Clasic;
-   * ``azr://<publisher>/<offer>/<sku>/<version>``, for Microsoft Azure;
+   * ``azr://<image-id>``, for Microsoft Azure Clasic (deprecated);
+   * ``azr://[region]/<publisher>/<offer>/<sku>/<version>``, for Microsoft Azure;
    * ``azr://[snapshots|disk]/<rgname>/<diskname>``, for Microsoft Azure;
    * ``<fedcloud_endpoint_url>/<image_id>``, for FedCloud OCCI connector.
    * ``appdb://<site_name>/<apc_name>?<vo_name>``, for FedCloud OCCI, OpenStack or EGI connectors using AppDB info (from vers. 1.6.0, 1.8.6 and 1.10.2 respectively).
-     In case of EGI connector the ``vo_name`` is not required as it will be get from auth data.
-   * ``appdb://<apc_name>?<vo_name>``, for FedCloud OCCI, OpenStack or EGI connectors without setting site_name.
+     In case of EGI connector the ``vo_name`` is not required as it will be get from auth data (deprecated use egi protocol instead).
+   * ``appdb://<apc_name>?<vo_name>``, for FedCloud OCCI, OpenStack or EGI connectors without setting site_name (deprecated use egi protocol instead).
+   * ``egi://<site_name>/<apc_name>?<vo_name>``, for OpenStack or EGI connectors using EGI IS info (from vers. 1.19.0).
    * ``docker://<docker_image>``, for Docker images.
    * ``fbw://<fns_server>/<image-id>``, for FogBow images.
    * ``lin://linode/<image-id>``, for Linode images.

test/unit/FedcloudInfo.py

@@ -10,6 +10,7 @@
 sys.path.append("..")
 sys.path.append(".")
 
+from radl.radl import system, Feature
 from IM.FedcloudInfo import FedcloudInfo
 from mock import patch, MagicMock
 
@@ -58,21 +59,23 @@ def get_response(self, method, url, **kwargs):
                 resp.status_code = 200
                 resp.json.return_value = [
                     {
-                        "egi_id": "egi.docker",
-                        "id": "image_id2",
+                        "egi_id": "egi_vm_images/docker:22.04",
+                        "id": "93759c63-b998-4222-84eb-5fa10af1113b",
                         "endpoint": "https://keystone.recas.ba.infn.it/v3",
-                        "mpuri": "https://appdb.egi.eu/store/vo/image/0c0a1ffc-b936-5efd-920c-b648a02cccf4:13976/",
-                        "name": "EGI Docker",
-                        "version": "2023.09.19",
+                        "mpuri": ("registry.egi.eu/egi_vm_images/docker:22.04-sha256:"
+                                  "09ef8b577905758d51bc2515f8bef89a9ee11202b64878bb59a02417b55c95ce"),
+                        "name": "registry.egi.eu egi_vm_images/docker:22.04",
+                        "version": "2025-10-07-fc45d0d3",
                         "vo": "fedcloud.egi.eu",
                     },
                     {
-                        "egi_id": "egi.docker",
-                        "id": "image_id3",
+                        "egi_id": "egi_vm_images/ubuntu:24.04",
+                        "id": "8d4755c8-6157-4dd0-a60e-3f8cfbfcbc60",
                         "endpoint": "https://keystone.recas.ba.infn.it/v3",
-                        "mpuri": "https://appdb.egi.eu/store/vo/image/0c0a1ffc-b936-5e66-920c-b648a02cccf4:13976/",
-                        "name": "EGI Docker",
-                        "version": "2023.09.01",
+                        "mpuri": ("registry.egi.eu/egi_vm_images/ubuntu:24.04-sha256:"
+                                  "4bcb811d1fa2f53e794d9be240c36f004e45308128b2cba9e3a51a7975bd2359"),
+                        "name": "registry.egi.eu egi_vm_images/ubuntu:24.04",
+                        "version": "2025-10-07-fc45d0d3",
                         "vo": "fedcloud.egi.eu",
                     },
                 ]
@@ -88,17 +91,17 @@ def test_get_site_url(self, requests):
     def test_get_image_id(self, requests):
         requests.side_effect = self.get_response
         res = FedcloudInfo.get_image_id(
-            "INFN-CLOUD-BARI", "egi.docker", "fedcloud.egi.eu"
+            "INFN-CLOUD-BARI", "egi_vm_images/ubuntu:24.04", "fedcloud.egi.eu"
         )
-        self.assertEqual(res, "image_id2")
+        self.assertEqual(res, "8d4755c8-6157-4dd0-a60e-3f8cfbfcbc60")
 
     @patch("requests.request")
     def test_get_image_data(self, requests):
         requests.side_effect = self.get_response
-        str_url = "egi://INFN-CLOUD-BARI/egi.docker?fedcloud.egi.eu"
+        str_url = "egi://INFN-CLOUD-BARI/egi_vm_images/ubuntu:24.04?fedcloud.egi.eu"
         site_url, image_id, _ = FedcloudInfo.get_image_data(str_url)
         self.assertEqual(site_url, "https://keystone.recas.ba.infn.it")
-        self.assertEqual(image_id, "image_id2")
+        self.assertEqual(image_id, "8d4755c8-6157-4dd0-a60e-3f8cfbfcbc60")
 
     @patch("requests.request")
     def test_get_project_ids(self, requests):
@@ -107,3 +110,16 @@ def test_get_project_ids(self, requests):
         self.assertEqual(
             projects, {"ops": "projectid1", "vo.access.egi.eu": "projectid2"}
         )
+
+    @patch("requests.request")
+    def test_search_vm(self, requests):
+        radl_system = system("s0", [Feature("disk.0.os.name", "=", "linux"),
+                                    Feature("disk.0.os.flavour", "=", "ubuntu"),
+                                    Feature("disk.0.os.version", "=", "24.04")])
+        requests.side_effect = self.get_response
+        res = FedcloudInfo.search_vm(radl_system)
+
+        self.assertEqual(len(res), 1)
+        self.assertEqual(res[0].getValue("disk.0.image.url"),
+                         "ost://keystone.recas.ba.infn.it/8d4755c8-6157-4dd0-a60e-3f8cfbfcbc60")
+        self.assertEqual(res[0].getValue("disk.0.image.vo"), "fedcloud.egi.eu")

test/unit/test_im_logic.py

@@ -1407,9 +1407,9 @@ def test_get_cloud_info(self, appdbis):
 
     @patch('IM.InfrastructureManager.VMRC')
     @patch('IM.InfrastructureManager.FedcloudInfo')
-    def test_systems_with_iis(self, appdbis, vmrc):
+    def test_systems_with_iis(self, egiis, vmrc):
         auth = self.getAuth([0], [0])
-        auth.auth_list.append({"type": "AppDBIS", "host": "http://is.marie.hellasgrid.gr"})
+        auth.auth_list.append({"type": "EGIIS", "host": "https://is.cloud.egi.eu"})
         radl = RADL()
         radl.add(system("s0", [Feature("disk.0.os.name", "=", "linux"),
                                Feature("disk.0.os.flavour", "=", "ubuntu"),
@@ -1418,13 +1418,13 @@ def test_systems_with_iis(self, appdbis, vmrc):
         inf = InfrastructureInfo()
         inf.id = "1"
 
-        sys_appdb = system("s0", [Feature("disk.0.os.name", "=", "linux"),
-                                  Feature("disk.0.os.flavour", "=", "ubuntu"),
-                                  Feature("disk.0.os.version", "=", "18.04"),
-                                  Feature("disk.0.image.url", "=", "https://fedcloud.eu"),
-                                  Feature("disk.0.image.vo", "=", "fedcloud.egi.eu")])
+        sys_egi = system("s0", [Feature("disk.0.os.name", "=", "linux"),
+                                Feature("disk.0.os.flavour", "=", "ubuntu"),
+                                Feature("disk.0.os.version", "=", "18.04"),
+                                Feature("disk.0.image.url", "=", "https://fedcloud.eu"),
+                                Feature("disk.0.image.vo", "=", "fedcloud.egi.eu")])
 
-        appdbis.search_vm.return_value = [sys_appdb]
+        egiis.search_vm.return_value = [sys_egi]
 
         sys_vmrc = system("s0", [Feature("disk.0.os.name", "=", "linux"),
                                  Feature("disk.0.os.flavour", "=", "ubuntu"),