Merge pull request #269 from QuanMPhm/262/cluster_name

Add internal cluster name as a resource attribute

Author: knikolla

src/coldfront_plugin_cloud/attributes.py

@@ -34,6 +34,7 @@ class CloudAllocationAttribute:
 RESOURCE_DEFAULT_NETWORK_CIDR = "OpenStack Default Network CIDR"
 
 RESOURCE_EULA_URL = "EULA URL"
+RESOURCE_CLUSTER_NAME = "Internal Cluster Name"
 
 RESOURCE_ATTRIBUTES = [
     CloudResourceAttribute(name=RESOURCE_AUTH_URL),
@@ -48,6 +49,7 @@ class CloudAllocationAttribute:
     CloudResourceAttribute(name=RESOURCE_EULA_URL),
     CloudResourceAttribute(name=RESOURCE_DEFAULT_PUBLIC_NETWORK),
     CloudResourceAttribute(name=RESOURCE_DEFAULT_NETWORK_CIDR),
+    CloudResourceAttribute(name=RESOURCE_CLUSTER_NAME),
 ]
 
 # TODO: Migration to rename the OpenStack specific prefix out of these attrs

src/coldfront_plugin_cloud/management/commands/add_openshift_resource.py

@@ -24,6 +24,12 @@ def add_arguments(self, parser):
         parser.add_argument(
             "--name", type=str, required=True, help="Name of OpenShift resource"
         )
+        parser.add_argument(
+            "--internal-name",
+            type=str,
+            required=False,
+            help="Internal name of cluster used for invoicing. Defaults to public name",
+        )
         parser.add_argument(
             "--api-url",
             type=str,
@@ -99,3 +105,12 @@ def handle(self, *args, **options):
             resource=openshift,
             value="true" if options["ibm_storage_available"] else "false",
         )
+        ResourceAttribute.objects.get_or_create(
+            resource_attribute_type=ResourceAttributeType.objects.get(
+                name=attributes.RESOURCE_CLUSTER_NAME
+            ),
+            resource=openshift,
+            value=options["internal_name"]
+            if options["internal_name"]
+            else options["name"],
+        )

src/coldfront_plugin_cloud/management/commands/add_openstack_resource.py

@@ -17,6 +17,12 @@ def add_arguments(self, parser):
         parser.add_argument(
             "--name", type=str, required=True, help="Name of OpenStack resource"
         )
+        parser.add_argument(
+            "--internal-name",
+            type=str,
+            required=False,
+            help="Internal name of cluster used for invoicing. Defaults to public name",
+        )
         parser.add_argument(
             "--auth-url",
             type=str,
@@ -133,6 +139,15 @@ def handle(self, *args, **options):
             resource=openstack,
             value=options["role"],
         )
+        ResourceAttribute.objects.get_or_create(
+            resource_attribute_type=ResourceAttributeType.objects.get(
+                name=attributes.RESOURCE_CLUSTER_NAME
+            ),
+            resource=openstack,
+            value=options["internal_name"]
+            if options["internal_name"]
+            else options["name"],
+        )
 
         # Quantity values do not make sense for an ESI allocation
         if not options["esi"]:

src/coldfront_plugin_cloud/management/commands/calculate_storage_gb_hours.py

@@ -20,12 +20,6 @@
 
 _RATES = None
 
-RESOURCE_NAME_TO_NERC_SERVICE = {
-    "NERC": "stack",
-    "NERC-OCP": "ocp-prod",
-    "NERC-OCP-EDU": "academic",
-}
-
 
 def get_rates():
     # nerc-rates doesn't work with Python 3.9, which is what ColdFront is currently
@@ -197,23 +191,22 @@ def upload_to_s3(s3_endpoint, s3_bucket, file_location, invoice_month):
         logger.info(f"Uploaded to {secondary_location}.")
 
     def handle(self, *args, **options):
-        def get_outages_for_service(resource_name: str):
+        def get_outages_for_service(cluster_name: str):
             """Get outages for a service from nerc-rates.
 
-            :param resource_name: Name of the resource to get outages for.
+            :param cluster_name: Name of the cluster to get outages for.
             :return: List of excluded intervals or None.
             """
-            service_name = RESOURCE_NAME_TO_NERC_SERVICE.get(resource_name)
-            if service_name:
-                return utils.load_outages_from_nerc_rates(
-                    options["start"], options["end"], service_name
-                )
-            return None
+            return utils.load_outages_from_nerc_rates(
+                options["start"], options["end"], cluster_name
+            )
 
         def process_invoice_row(allocation, attrs, su_name, rate):
             """Calculate the value and write the bill using the writer."""
-            resource_name = allocation.resources.first().name
-            excluded_intervals_list = get_outages_for_service(resource_name)
+            internal_cluster_name = allocation.resources.first().get_attribute(
+                attributes.RESOURCE_CLUSTER_NAME
+            )
+            excluded_intervals_list = get_outages_for_service(internal_cluster_name)
 
             time = 0
             for attribute in attrs:
@@ -234,7 +227,7 @@ def process_invoice_row(allocation, attrs, su_name, rate):
                         attributes.ALLOCATION_PROJECT_ID
                     ),
                     PI=allocation.project.pi.email,
-                    Cluster_Name=allocation.resources.first().name,
+                    Cluster_Name=internal_cluster_name,
                     Institution_Specific_Code=allocation.get_attribute(
                         attributes.ALLOCATION_INSTITUTION_SPECIFIC_CODE
                     )

src/coldfront_plugin_cloud/tests/base.py

@@ -63,12 +63,15 @@ def new_esi_resource(name=None, auth_url=None) -> Resource:
         return Resource.objects.get(name=resource_name)
 
     @staticmethod
-    def new_openstack_resource(name=None, auth_url=None) -> Resource:
+    def new_openstack_resource(
+        name=None, internal_name=None, auth_url=None
+    ) -> Resource:
         resource_name = name or uuid.uuid4().hex
 
         call_command(
             "add_openstack_resource",
             name=resource_name,
+            internal_name=internal_name,
             auth_url=auth_url or f"https://{resource_name}/identity/v3",
             projects_domain="default",
             users_domain="default",

src/coldfront_plugin_cloud/tests/unit/test_attribute_migration.py

@@ -124,7 +124,9 @@ def test_rename_identity_url(self):
                     new_resource_attrs,
                 ):
                     call_command("register_cloud_attributes")
-                    resource = self.new_openstack_resource("Example", auth_url_val)
+                    resource = self.new_openstack_resource(
+                        "Example", auth_url=auth_url_val
+                    )
 
                     self.assertEqual(
                         resource.get_attribute(new_auth_url_name),

src/coldfront_plugin_cloud/tests/unit/test_calculate_quota_unit_hours.py

@@ -92,10 +92,6 @@ def test_new_allocation_quota(self, mock_load_outages):
                 "2020-03",
             )
 
-        # Verify that load_outages_from_nerc_rates is not called when resource name
-        # doesn't match NERC service mapping
-        mock_load_outages.assert_not_called()
-
     def test_new_allocation_quota_expired(self):
         """Test that expiration doesn't affect invoicing."""
         self.resource = self.new_openshift_resource(
@@ -607,10 +603,6 @@ def test_load_excluded_intervals_invalid(self):
         with self.assertRaises(AssertionError):
             utils.load_excluded_intervals(invalid_interval)
 
-    @patch(
-        "coldfront_plugin_cloud.management.commands.calculate_storage_gb_hours.RESOURCE_NAME_TO_NERC_SERVICE",
-        {"TEST-RESOURCE": "test-service"},
-    )
     @patch(
         "coldfront_plugin_cloud.management.commands.calculate_storage_gb_hours.get_rates"
     )
@@ -633,7 +625,9 @@ def test_nerc_outages_integration(self, mock_rates_loader):
             with freezegun.freeze_time("2020-03-01"):
                 user = self.new_user()
                 project = self.new_project(pi=user)
-                resource = self.new_openstack_resource(name="TEST-RESOURCE")
+                resource = self.new_openstack_resource(
+                    name="TEST-RESOURCE", internal_name="test-service"
+                )
                 allocation = self.new_allocation(project, resource, 100)
                 for attr, val in [
                     (attributes.ALLOCATION_PROJECT_NAME, "test"),