Skip to content

Commit 2576528

Browse files
XN137XN137
committed
Pass AccessConfig into FileIOFactory
it should not be the responsibility of the `FileIOFactory` to know how to infer the `AccessConfig`
1 parent 53255dd commit 2576528

21 files changed

+105
-194
lines changed

‎runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java‎

Lines changed: 10 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,6 @@
108108
import org.apache.polaris.core.exceptions.CommitConflictException;
109109
import org.apache.polaris.core.persistence.PolarisMetaStoreManager;
110110
import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper;
111-
import org.apache.polaris.core.persistence.ResolvedPolarisEntity;
112111
import org.apache.polaris.core.persistence.dao.entity.BaseResult;
113112
import org.apache.polaris.core.persistence.dao.entity.DropEntityResult;
114113
import org.apache.polaris.core.persistence.dao.entity.EntityResult;
@@ -120,12 +119,14 @@
120119
import org.apache.polaris.core.persistence.resolver.ResolverFactory;
121120
import org.apache.polaris.core.persistence.resolver.ResolverPath;
122121
import org.apache.polaris.core.persistence.resolver.ResolverStatus;
122+
import org.apache.polaris.core.storage.AccessConfig;
123123
import org.apache.polaris.core.storage.PolarisStorageActions;
124124
import org.apache.polaris.core.storage.StorageLocation;
125125
import org.apache.polaris.core.storage.StorageUtil;
126126
import org.apache.polaris.service.catalog.SupportsNotifications;
127127
import org.apache.polaris.service.catalog.common.CatalogUtils;
128128
import org.apache.polaris.service.catalog.common.LocationUtils;
129+
import org.apache.polaris.service.catalog.io.AccessConfigProvider;
129130
import org.apache.polaris.service.catalog.io.FileIOFactory;
130131
import org.apache.polaris.service.catalog.io.FileIOUtil;
131132
import org.apache.polaris.service.catalog.validation.IcebergPropertiesValidation;
@@ -178,6 +179,7 @@ public class IcebergCatalog extends BaseMetastoreViewCatalog
178179
private long catalogId = -1;
179180
private String defaultBaseLocation;
180181
private Map<String, String> catalogProperties;
182+
private final AccessConfigProvider accessConfigProvider;
181183
private FileIOFactory fileIOFactory;
182184
private PolarisMetaStoreManager metaStoreManager;
183185

@@ -195,6 +197,7 @@ public IcebergCatalog(
195197
PolarisResolutionManifestCatalogView resolvedEntityView,
196198
SecurityContext securityContext,
197199
TaskExecutor taskExecutor,
200+
AccessConfigProvider accessConfigProvider,
198201
FileIOFactory fileIOFactory,
199202
PolarisEventListener polarisEventListener) {
200203
this.diagnostics = diagnostics;
@@ -207,6 +210,7 @@ public IcebergCatalog(
207210
this.taskExecutor = taskExecutor;
208211
this.catalogId = catalogEntity.getId();
209212
this.catalogName = catalogEntity.getName();
213+
this.accessConfigProvider = accessConfigProvider;
210214
this.fileIOFactory = fileIOFactory;
211215
this.metaStoreManager = metaStoreManager;
212216
this.polarisEventListener = polarisEventListener;
@@ -2076,16 +2080,16 @@ private FileIO loadFileIOForTableLike(
20762080
PolarisResolvedPathWrapper resolvedStorageEntity,
20772081
Map<String, String> tableProperties,
20782082
Set<PolarisStorageActions> storageActions) {
2079-
// Reload fileIO based on table specific context
2080-
FileIO fileIO =
2081-
fileIOFactory.loadFileIO(
2083+
AccessConfig accessConfig =
2084+
accessConfigProvider.getAccessConfig(
20822085
callContext,
2083-
ioImplClassName,
2084-
tableProperties,
20852086
identifier,
20862087
readLocations,
20872088
storageActions,
2089+
Optional.empty(),
20882090
resolvedStorageEntity);
2091+
// Reload fileIO based on table specific context
2092+
FileIO fileIO = fileIOFactory.loadFileIO(accessConfig, ioImplClassName, tableProperties);
20892093
// ensure the new fileIO is closed when the catalog is closed
20902094
closeableGroup.addCloseable(fileIO);
20912095
return fileIO;
@@ -2595,26 +2599,6 @@ private Page<TableIdentifier> listTableLike(
25952599
.map(record -> TableIdentifier.of(parentNamespace, record.getName()));
25962600
}
25972601

2598-
/**
2599-
* Load FileIO with provided impl and properties
2600-
*
2601-
* @param ioImpl full class name of a custom FileIO implementation
2602-
* @param properties used to initialize the FileIO implementation
2603-
* @return FileIO object
2604-
*/
2605-
protected FileIO loadFileIO(String ioImpl, Map<String, String> properties) {
2606-
IcebergTableLikeEntity icebergTableLikeEntity = IcebergTableLikeEntity.of(catalogEntity);
2607-
TableIdentifier identifier = icebergTableLikeEntity.getTableIdentifier();
2608-
Set<String> locations = Set.of(catalogEntity.getBaseLocation());
2609-
ResolvedPolarisEntity resolvedCatalogEntity =
2610-
new ResolvedPolarisEntity(catalogEntity, List.of(), List.of());
2611-
PolarisResolvedPathWrapper resolvedPath =
2612-
new PolarisResolvedPathWrapper(List.of(resolvedCatalogEntity));
2613-
Set<PolarisStorageActions> storageActions = Set.of(PolarisStorageActions.ALL);
2614-
return fileIOFactory.loadFileIO(
2615-
callContext, ioImpl, properties, identifier, locations, storageActions, resolvedPath);
2616-
}
2617-
26182602
private int getMaxMetadataRefreshRetries() {
26192603
return realmConfig.getConfig(FeatureConfiguration.MAX_METADATA_REFRESH_RETRIES);
26202604
}

‎runtime/service/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java‎

Lines changed: 4 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -25,55 +25,32 @@
2525
import jakarta.inject.Inject;
2626
import java.util.HashMap;
2727
import java.util.Map;
28-
import java.util.Optional;
29-
import java.util.Set;
3028
import org.apache.iceberg.CatalogUtil;
31-
import org.apache.iceberg.catalog.TableIdentifier;
3229
import org.apache.iceberg.io.FileIO;
33-
import org.apache.polaris.core.context.CallContext;
34-
import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper;
3530
import org.apache.polaris.core.storage.AccessConfig;
36-
import org.apache.polaris.core.storage.PolarisStorageActions;
3731

3832
/**
3933
* A default FileIO factory implementation for creating Iceberg {@link FileIO} instances with
4034
* contextual table-level properties.
4135
*
4236
* <p>This class acts as a translation layer between Polaris properties and the properties required
43-
* by Iceberg's {@link FileIO}. For example, it evaluates storage actions and retrieves subscoped
44-
* credentials to initialize a {@link FileIO} instance with the most limited permissions necessary.
37+
* by Iceberg's {@link FileIO}.
4538
*/
4639
@ApplicationScoped
4740
@Identifier("default")
4841
public class DefaultFileIOFactory implements FileIOFactory {
4942

50-
private final AccessConfigProvider accessConfigProvider;
51-
5243
@Inject
53-
public DefaultFileIOFactory(AccessConfigProvider accessConfigProvider) {
54-
this.accessConfigProvider = accessConfigProvider;
55-
}
44+
public DefaultFileIOFactory() {}
5645

5746
@Override
5847
public FileIO loadFileIO(
59-
@Nonnull CallContext callContext,
48+
@Nonnull AccessConfig accessConfig,
6049
@Nonnull String ioImplClassName,
61-
@Nonnull Map<String, String> properties,
62-
@Nonnull TableIdentifier identifier,
63-
@Nonnull Set<String> tableLocations,
64-
@Nonnull Set<PolarisStorageActions> storageActions,
65-
@Nonnull PolarisResolvedPathWrapper resolvedEntityPath) {
50+
@Nonnull Map<String, String> properties) {
6651

6752
// Get subcoped creds
6853
properties = new HashMap<>(properties);
69-
AccessConfig accessConfig =
70-
accessConfigProvider.getAccessConfig(
71-
callContext,
72-
identifier,
73-
tableLocations,
74-
storageActions,
75-
Optional.empty(),
76-
resolvedEntityPath);
7754

7855
// Update the FileIO with the subscoped credentials
7956
// Update with properties in case there are table-level overrides the credentials should

‎runtime/service/src/main/java/org/apache/polaris/service/catalog/io/FileIOFactory.java‎

Lines changed: 4 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,8 @@
2121
import jakarta.annotation.Nonnull;
2222
import jakarta.enterprise.context.ApplicationScoped;
2323
import java.util.Map;
24-
import java.util.Set;
25-
import org.apache.iceberg.catalog.TableIdentifier;
2624
import org.apache.iceberg.io.FileIO;
27-
import org.apache.polaris.core.context.CallContext;
28-
import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper;
29-
import org.apache.polaris.core.storage.PolarisStorageActions;
25+
import org.apache.polaris.core.storage.AccessConfig;
3026

3127
/**
3228
* Interface for providing a way to construct FileIO objects, such as for reading/writing S3.
@@ -41,21 +37,13 @@ public interface FileIOFactory {
4137
* <p>This method may obtain subscoped credentials to restrict the FileIO's permissions, ensuring
4238
* secure and limited access to the table's data and locations.
4339
*
44-
* @param callContext the call for which the FileIO is being loaded.
40+
* @param accessConfig the access configuration containing credentials and other properties.
4541
* @param ioImplClassName the class name of the FileIO implementation to load.
4642
* @param properties configuration properties for the FileIO.
47-
* @param identifier the table identifier.
48-
* @param tableLocations locations associated with the table.
49-
* @param storageActions storage actions allowed for the table.
50-
* @param resolvedEntityPath resolved paths for the entities.
5143
* @return a configured FileIO instance.
5244
*/
5345
FileIO loadFileIO(
54-
@Nonnull CallContext callContext,
46+
@Nonnull AccessConfig accessConfig,
5547
@Nonnull String ioImplClassName,
56-
@Nonnull Map<String, String> properties,
57-
@Nonnull TableIdentifier identifier,
58-
@Nonnull Set<String> tableLocations,
59-
@Nonnull Set<PolarisStorageActions> storageActions,
60-
@Nonnull PolarisResolvedPathWrapper resolvedEntityPath);
48+
@Nonnull Map<String, String> properties);
6149
}

‎runtime/service/src/main/java/org/apache/polaris/service/catalog/io/WasbTranslatingFileIOFactory.java‎

Lines changed: 6 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,8 @@
2323
import jakarta.enterprise.context.ApplicationScoped;
2424
import jakarta.inject.Inject;
2525
import java.util.Map;
26-
import java.util.Set;
27-
import org.apache.iceberg.catalog.TableIdentifier;
2826
import org.apache.iceberg.io.FileIO;
29-
import org.apache.polaris.core.context.CallContext;
30-
import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper;
31-
import org.apache.polaris.core.storage.PolarisStorageActions;
27+
import org.apache.polaris.core.storage.AccessConfig;
3228

3329
/** A {@link FileIOFactory} that translates WASB paths to ABFS ones */
3430
@ApplicationScoped
@@ -38,27 +34,16 @@ public class WasbTranslatingFileIOFactory implements FileIOFactory {
3834
private final FileIOFactory defaultFileIOFactory;
3935

4036
@Inject
41-
public WasbTranslatingFileIOFactory(AccessConfigProvider accessConfigProvider) {
42-
defaultFileIOFactory = new DefaultFileIOFactory(accessConfigProvider);
37+
public WasbTranslatingFileIOFactory() {
38+
defaultFileIOFactory = new DefaultFileIOFactory();
4339
}
4440

4541
@Override
4642
public FileIO loadFileIO(
47-
@Nonnull CallContext callContext,
43+
@Nonnull AccessConfig accessConfig,
4844
@Nonnull String ioImplClassName,
49-
@Nonnull Map<String, String> properties,
50-
@Nonnull TableIdentifier identifier,
51-
@Nonnull Set<String> tableLocations,
52-
@Nonnull Set<PolarisStorageActions> storageActions,
53-
@Nonnull PolarisResolvedPathWrapper resolvedEntityPath) {
45+
@Nonnull Map<String, String> properties) {
5446
return new WasbTranslatingFileIO(
55-
defaultFileIOFactory.loadFileIO(
56-
callContext,
57-
ioImplClassName,
58-
properties,
59-
identifier,
60-
tableLocations,
61-
storageActions,
62-
resolvedEntityPath));
47+
defaultFileIOFactory.loadFileIO(accessConfig, ioImplClassName, properties));
6348
}
6449
}

‎runtime/service/src/main/java/org/apache/polaris/service/context/catalog/PolarisCallContextCatalogFactory.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifest;
3434
import org.apache.polaris.core.persistence.resolver.ResolverFactory;
3535
import org.apache.polaris.service.catalog.iceberg.IcebergCatalog;
36+
import org.apache.polaris.service.catalog.io.AccessConfigProvider;
3637
import org.apache.polaris.service.catalog.io.FileIOFactory;
3738
import org.apache.polaris.service.events.listeners.PolarisEventListener;
3839
import org.apache.polaris.service.task.TaskExecutor;
@@ -46,6 +47,7 @@ public class PolarisCallContextCatalogFactory implements CallContextCatalogFacto
4647

4748
private final PolarisDiagnostics diagnostics;
4849
private final TaskExecutor taskExecutor;
50+
private final AccessConfigProvider accessConfigProvider;
4951
private final FileIOFactory fileIOFactory;
5052
private final ResolverFactory resolverFactory;
5153
private final MetaStoreManagerFactory metaStoreManagerFactory;
@@ -57,12 +59,14 @@ public PolarisCallContextCatalogFactory(
5759
ResolverFactory resolverFactory,
5860
MetaStoreManagerFactory metaStoreManagerFactory,
5961
TaskExecutor taskExecutor,
62+
AccessConfigProvider accessConfigProvider,
6063
FileIOFactory fileIOFactory,
6164
PolarisEventListener polarisEventListener) {
6265
this.diagnostics = diagnostics;
6366
this.resolverFactory = resolverFactory;
6467
this.metaStoreManagerFactory = metaStoreManagerFactory;
6568
this.taskExecutor = taskExecutor;
69+
this.accessConfigProvider = accessConfigProvider;
6670
this.fileIOFactory = fileIOFactory;
6771
this.polarisEventListener = polarisEventListener;
6872
}
@@ -89,6 +93,7 @@ public Catalog createCallContextCatalog(
8993
resolvedManifest,
9094
securityContext,
9195
taskExecutor,
96+
accessConfigProvider,
9297
fileIOFactory,
9398
polarisEventListener);
9499

‎runtime/service/src/main/java/org/apache/polaris/service/task/TaskFileIOSupplier.java‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import java.util.HashMap;
2424
import java.util.List;
2525
import java.util.Map;
26+
import java.util.Optional;
2627
import java.util.Set;
2728
import org.apache.iceberg.CatalogProperties;
2829
import org.apache.iceberg.catalog.TableIdentifier;
@@ -32,19 +33,25 @@
3233
import org.apache.polaris.core.entity.TaskEntity;
3334
import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper;
3435
import org.apache.polaris.core.persistence.ResolvedPolarisEntity;
36+
import org.apache.polaris.core.storage.AccessConfig;
3537
import org.apache.polaris.core.storage.PolarisStorageActions;
38+
import org.apache.polaris.service.catalog.io.AccessConfigProvider;
3639
import org.apache.polaris.service.catalog.io.FileIOFactory;
3740

3841
@ApplicationScoped
3942
public class TaskFileIOSupplier {
4043
private final FileIOFactory fileIOFactory;
44+
private final AccessConfigProvider accessConfigProvider;
4145

4246
@Inject
43-
public TaskFileIOSupplier(FileIOFactory fileIOFactory) {
47+
public TaskFileIOSupplier(
48+
FileIOFactory fileIOFactory, AccessConfigProvider accessConfigProvider) {
4449
this.fileIOFactory = fileIOFactory;
50+
this.accessConfigProvider = accessConfigProvider;
4551
}
4652

4753
public FileIO apply(TaskEntity task, TableIdentifier identifier, CallContext callContext) {
54+
4855
Map<String, String> internalProperties = task.getInternalPropertiesAsMap();
4956
Map<String, String> properties = new HashMap<>(internalProperties);
5057

@@ -55,11 +62,14 @@ public FileIO apply(TaskEntity task, TableIdentifier identifier, CallContext cal
5562
new ResolvedPolarisEntity(task, List.of(), List.of());
5663
PolarisResolvedPathWrapper resolvedPath =
5764
new PolarisResolvedPathWrapper(List.of(resolvedTaskEntity));
65+
AccessConfig accessConfig =
66+
accessConfigProvider.getAccessConfig(
67+
callContext, identifier, locations, storageActions, Optional.empty(), resolvedPath);
68+
5869
String ioImpl =
5970
properties.getOrDefault(
6071
CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.io.ResolvingFileIO");
6172

62-
return fileIOFactory.loadFileIO(
63-
callContext, ioImpl, properties, identifier, locations, storageActions, resolvedPath);
73+
return fileIOFactory.loadFileIO(accessConfig, ioImpl, properties);
6474
}
6575
}

‎runtime/service/src/test/java/org/apache/polaris/service/admin/PolarisAuthzTestBase.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -537,6 +537,7 @@ private void initBaseCatalog() {
537537
passthroughView,
538538
securityContext,
539539
Mockito.mock(),
540+
accessConfigProvider,
540541
fileIOFactory,
541542
polarisEventListener);
542543
this.baseCatalog.initialize(
@@ -562,17 +563,18 @@ protected TestPolarisCallContextCatalogFactory() {
562563
@Inject
563564
public TestPolarisCallContextCatalogFactory(
564565
PolarisDiagnostics diagnostics,
565-
StorageCredentialCache storageCredentialCache,
566566
ResolverFactory resolverFactory,
567567
MetaStoreManagerFactory metaStoreManagerFactory,
568568
TaskExecutor taskExecutor,
569+
AccessConfigProvider accessConfigProvider,
569570
FileIOFactory fileIOFactory,
570571
PolarisEventListener polarisEventListener) {
571572
super(
572573
diagnostics,
573574
resolverFactory,
574575
metaStoreManagerFactory,
575576
taskExecutor,
577+
accessConfigProvider,
576578
fileIOFactory,
577579
polarisEventListener);
578580
}

‎runtime/service/src/test/java/org/apache/polaris/service/admin/PolarisS3InteroperabilityTest.java‎

Lines changed: 3 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import java.util.HashMap;
2929
import java.util.List;
3030
import java.util.Map;
31+
import java.util.function.Supplier;
3132
import org.apache.iceberg.catalog.Namespace;
3233
import org.apache.iceberg.exceptions.ForbiddenException;
3334
import org.apache.iceberg.inmemory.InMemoryFileIO;
@@ -72,16 +73,8 @@ private static String makeTableLocation(
7273
}
7374

7475
public PolarisS3InteroperabilityTest() {
75-
TestServices.FileIOFactorySupplier fileIOFactorySupplier =
76-
(accessConfigProvider) ->
77-
(FileIOFactory)
78-
(callContext,
79-
ioImplClassName,
80-
properties,
81-
identifier,
82-
tableLocations,
83-
storageActions,
84-
resolvedEntityPath) -> new InMemoryFileIO();
76+
Supplier<FileIOFactory> fileIOFactorySupplier =
77+
() -> (FileIOFactory) (accessConfig, ioImplClassName, properties) -> new InMemoryFileIO();
8578
services =
8679
TestServices.builder()
8780
.config(SERVER_CONFIG)

0 commit comments

Comments
 (0)