Merge pull request #1734 from grycap/devel

Devel

Author: micafer

IM/connectors/Azure.py

@@ -558,11 +558,15 @@ def get_azure_vm_create_json(self, group_name, vm_name, nics, radl,
         # azr://Canonical/UbuntuServer/16.04.0-LTS/latest
         # azr://MicrosoftWindowsServerEssentials/WindowsServerEssentials/WindowsServerEssentials/latest
         image_values = (url[1] + url[2]).split("/")
-        if len(image_values) not in [3, 4]:
+        if len(image_values) not in [3, 4, 5]:
             raise Exception("The Azure image has to have the format: azr://publisher/offer/sku/version"
+                            " or azr://region/publisher/offer/sku/version"
                             " or azr://[snapshots|disk]/rgname/diskname")
 
         location = self.DEFAULT_LOCATION
+        if len(image_values) == 5:
+            location = image_values[0]
+            image_values = image_values[1:]
         if system.getValue('availability_zone'):
             location = system.getValue('availability_zone')
 
@@ -837,12 +841,6 @@ def create_vms(self, rg_name, inf, radl, requested_radl, num_vm, location,
         return vms
 
     def launch(self, inf, radl, requested_radl, num_vm, auth_data):
-        location = self.DEFAULT_LOCATION
-        if radl.systems[0].getValue('availability_zone'):
-            location = radl.systems[0].getValue('availability_zone')
-        else:
-            radl.systems[0].setValue('availability_zone', location)
-
         credentials, subscription_id = self.get_credentials(auth_data)
         compute_client = ComputeManagementClient(credentials, subscription_id)
 
@@ -852,12 +850,22 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data):
         # azr://Canonical/UbuntuServer/16.04.0-LTS/latest
         # azr://MicrosoftWindowsServerEssentials/WindowsServerEssentials/WindowsServerEssentials/latest
         image_values = (url[1] + url[2]).split("/")
-        if len(image_values) not in [3, 4]:
+        if len(image_values) not in [3, 4, 5]:
             raise Exception("The Azure image has to have the format: azr://publisher/offer/sku/version"
+                            " or azr://region/publisher/offer/sku/version"
                             " or azr://[snapshots|disk|image]/rgname/diskname")
         if len(image_values) == 3 and image_values[0] not in ["snapshot", "disk"]:
             raise Exception("Incorrect image url: it must be snapshot or disk.")
 
+        location = self.DEFAULT_LOCATION
+        if len(image_values) == 5:
+            location = image_values[0]
+            image_values = image_values[1:]
+        if radl.systems[0].getValue('availability_zone'):
+            location = radl.systems[0].getValue('availability_zone')
+        else:
+            radl.systems[0].setValue('availability_zone', location)
+
         if len(image_values) == 4:
             offers = compute_client.virtual_machine_images.list(location,
                                                                 image_values[0],
@@ -1274,3 +1282,44 @@ def list_images(self, auth_data, filters=None):
             res.append({"uri": "azr://%s/%s/%s/latest" % (pub, offer, sku),
                                "name": name})
         return self._filter_images(res, filters)
+
+    def get_quotas(self, auth_data, region=None):
+        credentials, subscription_id = self.get_credentials(auth_data)
+        compute_client = ComputeManagementClient(credentials, subscription_id)
+        location = self.DEFAULT_LOCATION
+        try:
+            # Get the region from the auth data
+            location = auth_data.getAuthInfo(self.type)[0].get('region', location)
+        except Exception:
+            pass
+        if region:
+            location = region
+
+        # Initialize default values
+        quotas = {}
+
+        try:
+            usage_list = compute_client.usage.list(location)
+            for usage in usage_list:
+                name = usage.name.localized_value.lower()
+                if name == "total regional vcpus":
+                    quotas["cores"] = {}
+                    quotas["cores"]["used"] = usage.current_value
+                    quotas["cores"]["limit"] = usage.limit
+                elif name == "virtual machines":
+                    quotas["instances"] = {}
+                    quotas["instances"]["used"] = usage.current_value
+                    quotas["instances"]["limit"] = usage.limit
+                elif "storage" in name and "disks" in name:
+                    quotas["volumes"] = {}
+                    quotas["volumes"]["used"] = usage.current_value
+                    quotas["volumes"]["limit"] = usage.limit
+                elif "family vcpus" in name:
+                    fam = usage.name.localized_value[:-13].strip().replace(" ", "_")
+                    quotas[fam] = {"cores": {}}
+                    quotas[fam]["cores"]["used"] = usage.current_value
+                    quotas[fam]["cores"]["limit"] = usage.limit
+            return quotas
+        except Exception:
+            self.log_exception("Error retrieving Azure quotas")
+            return {}

IM/connectors/CloudConnector.py

@@ -361,12 +361,13 @@ def _filter_images(self, image_list, filters=None):
                 res.append(image)
         return res
 
-    def get_quotas(self, auth_data):
+    def get_quotas(self, auth_data, region=None):
         """
         Get the number of used and available resources in the cloud provider
 
         Arguments:
           - auth_data(:py:class:`dict` of str objects): Authentication data to access cloud provider.
+          - region(str): Region to get the quotas. If None, the default region is used.
 
         Returns: dict with the following structure (if there are no limit in some metric, value is set to 1):
                     {

IM/connectors/Dummy.py

@@ -101,7 +101,7 @@ def list_images(self, auth_data, filters=None):
         return [{"uri": "mock0://linux.for.ev.er/image1", "name": "Image Name1"},
                 {"uri": "mock0://linux.for.ev.er/image2", "name": "Image Name2"}]
 
-    def get_quotas(self, auth_data):
+    def get_quotas(self, auth_data, region=None):
         return {"cores": {"used": 1, "limit": 10},
                 "ram": {"used": 1, "limit": 10},
                 "instances": {"used": 1, "limit": 10},

IM/connectors/OpenNebula.py

@@ -1308,7 +1308,7 @@ def _get_public_ip_quota(net_quotas, net_info):
 
         return res
 
-    def get_quotas(self, auth_data):
+    def get_quotas(self, auth_data, region=None):
         server = ServerProxy(self.server_url, allow_none=True)
         session_id = self.getSessionID(auth_data)
 

IM/connectors/OpenStack.py

@@ -2187,9 +2187,12 @@ def _get_tenant_id(auth):
 
         return None
 
-    def get_quotas(self, auth_data):
+    def get_quotas(self, auth_data, region=None):
         driver = self.get_driver(auth_data)
         tenant_id = self._get_tenant_id(auth_data.getAuthInfo(self.type, self.cloud.server)[0])
+        if region:
+            # In this case the region parameter refers to the tenant_id
+            tenant_id = region
         quotas = driver.ex_get_quota_set(tenant_id)
         try:
             net_quotas = driver.ex_get_network_quotas(tenant_id)

doc/source/tosca.rst

@@ -286,14 +286,14 @@ and, optionally, an availability zone.
 
     policies:
       - deploy_group_on_cloudid:
-        type: tosca.policies.indigo.Placement
-        properties: { cloud_id: cloudid1 }
-        targets: [ my_placement_group ]
+          type: tosca.policies.indigo.Placement
+          properties: { cloud_id: cloudid1 }
+          targets: [ my_placement_group ]
 
       - deploy_on_cloudid:
-        type: tosca.policies.indigo.Placement
-        properties: { cloud_id: cloudid2, availability_zone: some_zone }
-        targets: [ compute_three ]
+          type: tosca.policies.indigo.Placement
+          properties: { cloud_id: cloudid2, availability_zone: some_zone }
+          targets: [ compute_three ]
 
     ...
 
@@ -419,4 +419,30 @@ value of a internally defined value.
 
       outputs:
         node_ip:
-          value: { get_attribute: [ front, ansible_output, lrms_front_end_front_conf_front, tasks, 'grycap.nomad : nomad_secret_id', output ] }
+          value:
+            get_attribute: 
+              - front
+              - ansible_output
+              - lrms_front_end_front_conf_front
+              - tasks
+              - 'grycap.nomad : nomad_secret_id'
+              - output
+
+
+Random Input values
+^^^^^^^^^^^^^^^^^^^^
+
+The IM TOSCA parser supports the generation of random values for string inputs.
+The special string ``random(N)`` generates a random string of length N with
+alphanumeric characters. It can be used to define passwords or any other random
+string input value (from version 1.19.2).
+
+.. code-block:: yaml
+
+    ...
+
+      inputs:
+        app_password:
+          type: string
+          description: Password for the App
+          default: 'random(12)'

test/unit/connectors/Azure.py

@@ -699,6 +699,44 @@ def test_invalid_rg_name_launch(self, save_data, credentials, compute_client, re
         res = azure_cloud.launch_with_retry(inf, radl, radl, 1, auth, 1, 0)
         self.assertEqual(res, [(False, 'Attempt 1: Error: Invalid rg_name. It must be unique per infrastructure.\n')])
 
+    @patch('IM.connectors.Azure.ComputeManagementClient')
+    @patch('IM.connectors.Azure.ClientSecretCredential')
+    def test_get_quotas(self, credentials, compute_client):
+        auth = Authentication([{'id': 'azure', 'type': 'Azure', 'subscription_id': 'subscription_id',
+                                'client_id': 'client', 'secret': 'password', 'tenant': 'tenant'}])
+        azure_cloud = self.get_azure_cloud()
+
+        cclient = MagicMock()
+        compute_client.return_value = cclient
+        name1 = MagicMock(localized_value='Total Regional vCPUs')
+        name2 = MagicMock(localized_value='Standard DSv3 Family vCPUs')
+        name3 = MagicMock(localized_value='Standard Dv3 Family vCPUs')
+        name4 = MagicMock(localized_value='Standard Storage Managed Disks')
+        name5 = MagicMock(localized_value='Virtual Machines')
+        elem1 = MagicMock(limit=10, current_value=5)
+        elem2 = MagicMock(limit=8, current_value=4)
+        elem3 = MagicMock(limit=6, current_value=2)
+        elem4 = MagicMock(limit=20, current_value=10)
+        elem5 = MagicMock(limit=20, current_value=1)
+        elem1.name = name1
+        elem2.name = name2
+        elem3.name = name3
+        elem4.name = name4
+        elem5.name = name5
+        cclient.usage.list.return_value = [
+            elem1, elem2, elem3, elem4, elem5
+        ]
+
+        expected_quotas = {
+            'cores': {'used': 5, 'limit': 10},
+            'Standard_DSv3': {'cores': {'used': 4, 'limit': 8}},
+            'Standard_Dv3': {'cores': {'used': 2, 'limit': 6}},
+            'volumes': {'used': 10, 'limit': 20},
+            'instances': {'used': 1, 'limit': 20}
+        }
+        quotas = azure_cloud.get_quotas(auth, "northeurope")
+        self.assertEqual(quotas, expected_quotas)
+
 
 if __name__ == '__main__':
     unittest.main()