|
| 1 | +from typing import Optional, List |
| 2 | +from uuid import UUID |
| 3 | +from sempy._utils._log import log |
| 4 | +import pandas as pd |
| 5 | +from sempy_labs._helper_functions import ( |
| 6 | + resolve_lakehouse_name_and_id, |
| 7 | + resolve_workspace_id, |
| 8 | + resolve_lakehouse_id, |
| 9 | + _create_dataframe, |
| 10 | + _base_api, |
| 11 | + resolve_workspace_name_and_id, |
| 12 | +) |
| 13 | +import sempy_labs._icons as icons |
| 14 | + |
| 15 | + |
| 16 | +@log |
| 17 | +def list_schemas( |
| 18 | + lakehouse: Optional[str | UUID] = None, workspace: Optional[str | UUID] = None |
| 19 | +) -> pd.DataFrame: |
| 20 | + """ |
| 21 | + Lists the schemas within a Fabric lakehouse. |
| 22 | +
|
| 23 | + Parameters |
| 24 | + ---------- |
| 25 | + lakehouse : str | uuid.UUID, default=None |
| 26 | + The Fabric lakehouse name or ID. |
| 27 | + Defaults to None which resolves to the lakehouse attached to the notebook. |
| 28 | + workspace : str | uuid.UUID, default=None |
| 29 | + The Fabric workspace name or ID used by the lakehouse. |
| 30 | + Defaults to None which resolves to the workspace of the attached lakehouse |
| 31 | + or if no lakehouse attached, resolves to the workspace of the notebook. |
| 32 | +
|
| 33 | + Returns |
| 34 | + ------- |
| 35 | + pandas.DataFrame |
| 36 | + Shows the schemas within a lakehouse. |
| 37 | + """ |
| 38 | + |
| 39 | + columns = { |
| 40 | + "Schema Name": "str", |
| 41 | + } |
| 42 | + df = _create_dataframe(columns=columns) |
| 43 | + workspace_id = resolve_workspace_id(workspace) |
| 44 | + item_id = resolve_lakehouse_id(lakehouse, workspace) |
| 45 | + response = _base_api( |
| 46 | + request=f"{workspace_id}/{item_id}/api/2.1/unity-catalog/schemas?catalog_name={item_id}", |
| 47 | + client="onelake", |
| 48 | + ) |
| 49 | + |
| 50 | + rows = [] |
| 51 | + for s in response.json().get("schemas", []): |
| 52 | + rows.append( |
| 53 | + { |
| 54 | + "Schema Name": s.get("name", None), |
| 55 | + } |
| 56 | + ) |
| 57 | + |
| 58 | + if rows: |
| 59 | + df = pd.DataFrame(rows, columns=list(columns.keys())) |
| 60 | + |
| 61 | + return df |
| 62 | + |
| 63 | + |
| 64 | +def list_tables( |
| 65 | + lakehouse: Optional[str | UUID] = None, |
| 66 | + workspace: Optional[str | UUID] = None, |
| 67 | + schema: Optional[str | List[str]] = None, |
| 68 | +) -> pd.DataFrame: |
| 69 | + |
| 70 | + (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace) |
| 71 | + (item_name, item_id) = resolve_lakehouse_name_and_id(lakehouse, workspace) |
| 72 | + |
| 73 | + response = _base_api(f"/v1/workspaces/{workspace_id}/lakehouses/{item_id}") |
| 74 | + default_schema = response.json().get("properties", {}).get("defaultSchema", None) |
| 75 | + schema_enabled = True if default_schema else False |
| 76 | + |
| 77 | + columns = { |
| 78 | + "Workspace Name": "str", |
| 79 | + "Lakehouse Name": "str", |
| 80 | + "Table Name": "str", |
| 81 | + "Schema Name": "str", |
| 82 | + "Format": "str", |
| 83 | + "Type": "str", |
| 84 | + "Location": "str", |
| 85 | + } |
| 86 | + df = _create_dataframe(columns=columns) |
| 87 | + |
| 88 | + rows = [] |
| 89 | + if schema_enabled: |
| 90 | + schemas = list_schemas(lakehouse=lakehouse, workspace=workspace) |
| 91 | + if schema: |
| 92 | + if isinstance(schema, str): |
| 93 | + schema = [schema] |
| 94 | + schemas = schemas[schemas["Schema Name"].isin(schema)] |
| 95 | + |
| 96 | + # Loop through schemas |
| 97 | + for _, r in schemas.iterrows(): |
| 98 | + schema_name = r["Schema Name"] |
| 99 | + response = _base_api( |
| 100 | + request=f"{workspace_id}/{item_id}/api/2.1/unity-catalog/tables?catalog_name={item_id}&schema_name={schema_name}", |
| 101 | + client="onelake", |
| 102 | + ) |
| 103 | + # Loop through tables |
| 104 | + for t in response.json().get("tables", []): |
| 105 | + location = t.get("storage_location", {}) |
| 106 | + location = f'abfss://{location.split(".microsoft.com/")[1]}' |
| 107 | + rows.append( |
| 108 | + { |
| 109 | + "Workspace Name": workspace_name, |
| 110 | + "Lakehouse Name": item_name, |
| 111 | + "Table Name": t.get("name", {}), |
| 112 | + "Schema Name": schema_name, |
| 113 | + "Format": t.get("data_source_format", {}).capitalize(), |
| 114 | + "Type": "Managed", |
| 115 | + "Location": location, |
| 116 | + } |
| 117 | + ) |
| 118 | + else: |
| 119 | + if schema: |
| 120 | + print( |
| 121 | + f"{icons.info} The schema parameter has been ignored as the '{item_name}' lakehouse within the '{workspace_name}' workspace has schemas disabled." |
| 122 | + ) |
| 123 | + responses = _base_api( |
| 124 | + request=f"v1/workspaces/{workspace_id}/lakehouses/{item_id}/tables", |
| 125 | + uses_pagination=True, |
| 126 | + client="fabric_sp", |
| 127 | + ) |
| 128 | + for r in responses: |
| 129 | + for i in r.get("data", []): |
| 130 | + rows.append( |
| 131 | + { |
| 132 | + "Workspace Name": workspace_name, |
| 133 | + "Lakehouse Name": item_name, |
| 134 | + "Schema Name": None, |
| 135 | + "Table Name": i.get("name"), |
| 136 | + "Format": i.get("format"), |
| 137 | + "Type": i.get("type"), |
| 138 | + "Location": i.get("location"), |
| 139 | + } |
| 140 | + ) |
| 141 | + |
| 142 | + if rows: |
| 143 | + df = pd.DataFrame(rows, columns=list(columns.keys())) |
| 144 | + |
| 145 | + return df |
| 146 | + |
| 147 | + |
| 148 | +def schema_exists( |
| 149 | + schema: str, |
| 150 | + lakehouse: Optional[str | UUID] = None, |
| 151 | + workspace: Optional[str | UUID] = None, |
| 152 | +) -> bool: |
| 153 | + """ |
| 154 | + Indicates whether the specified schema exists within a Fabric lakehouse. |
| 155 | +
|
| 156 | + Parameters |
| 157 | + ---------- |
| 158 | + schema : str |
| 159 | + The name of the schema. |
| 160 | + lakehouse : str | uuid.UUID, default=None |
| 161 | + The Fabric lakehouse name or ID. |
| 162 | + Defaults to None which resolves to the lakehouse attached to the notebook. |
| 163 | + workspace : str | uuid.UUID, default=None |
| 164 | + The Fabric workspace name or ID used by the lakehouse. |
| 165 | + Defaults to None which resolves to the workspace of the attached lakehouse |
| 166 | + or if no lakehouse attached, resolves to the workspace of the notebook. |
| 167 | +
|
| 168 | + Returns |
| 169 | + ------- |
| 170 | + bool |
| 171 | + Indicates whether the specified schema exists within the lakehouse. |
| 172 | + """ |
| 173 | + |
| 174 | + df = list_schemas(lakehouse=lakehouse, workspace=workspace) |
| 175 | + return schema in df["Schema Name"].values |
| 176 | + |
| 177 | + # (workspace_name, workspace_id) = resolve_workspace_name_and_id(workspace) |
| 178 | + # (item_name, item_id) = resolve_lakehouse_name_and_id(lakehouse, workspace) |
| 179 | + # response = _base_api( |
| 180 | + # request=f"{workspace_id}/{item_id}/api/2.1/unity-catalog/schemas/{schema}", |
| 181 | + # client="onelake", |
| 182 | + # method="head", |
| 183 | + # ) |
| 184 | + |
| 185 | + # response.json() |
0 commit comments