Skip to content

Commit 162869c

Browse files
committed
Reversing resolved inputs and outputs key and velue on NuItaipuFacet. dataset path is now the key and dataset name is now the value
1 parent f34e2e2 commit 162869c

File tree

3 files changed

+14
-10
lines changed

3 files changed

+14
-10
lines changed

integration/spark/shared/src/main/java/io/openlineage/spark/agent/facets/NuItaipuFacet.java

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import java.util.Map;
1212

1313
import static io.openlineage.spark.agent.util.NuFacetsUtils.getConfigValues;
14+
import static io.openlineage.spark.agent.util.NuFacetsUtils.keyToValue;
1415

1516
@Getter
1617
@Slf4j
@@ -27,7 +28,7 @@ public class NuItaipuFacet extends NuRunFacet{
2728

2829
public NuItaipuFacet(@NonNull SparkSession sparkSession) {
2930
super(Versions.OPEN_LINEAGE_PRODUCER_URI);
30-
this.resolvedInputs = getConfigValues(RESOLVED_INPUTS_PREFIX, sparkSession.conf());
31-
this.resolvedOutputs = getConfigValues(RESOLVED_OUTPUTS_PREFIX, sparkSession.conf());
31+
this.resolvedInputs = keyToValue(getConfigValues(RESOLVED_INPUTS_PREFIX, sparkSession.conf()));
32+
this.resolvedOutputs = keyToValue(getConfigValues(RESOLVED_OUTPUTS_PREFIX, sparkSession.conf()));
3233
}
3334
}

integration/spark/shared/src/main/java/io/openlineage/spark/agent/util/NuFacetsUtils.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,4 +40,9 @@ public static Map<String, String> getConfigValues(String prefix, RuntimeConfig c
4040
return new AbstractMap.SimpleEntry<>(datasetName, datasetPath);
4141
}).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
4242
}
43+
44+
public static Map<String, String> keyToValue(Map<String, String> map) {
45+
return map.entrySet().stream()
46+
.collect(Collectors.toMap(Map.Entry::getValue, Map.Entry::getKey));
47+
}
4348
}

integration/spark/shared/src/test/java/io/openlineage/spark/agent/facets/builder/NuFacetBuilderTest.java

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -92,16 +92,14 @@ void testBuildItaipuFacet() {
9292
});
9393

9494
Map<String, String> expectedResolvedInputs = new HashMap<>();
95-
expectedResolvedInputs.put("inputName_1", "s3a://path/to/inputName_1");
96-
expectedResolvedInputs.put("inputName_2", "s3a://path/to/inputName_2");
97-
expectedResolvedInputs.put("inputName_3", "s3a://path/to/inputName_3");
98-
95+
expectedResolvedInputs.put("s3a://path/to/inputName_1", "inputName_1");
96+
expectedResolvedInputs.put("s3a://path/to/inputName_2", "inputName_2");
97+
expectedResolvedInputs.put("s3a://path/to/inputName_3", "inputName_3");
9998

10099
Map<String, String> expectedResolvedOutputs = new HashMap<>();
101-
expectedResolvedOutputs.put("outputName_1", "s3a://path/to/outputName_1");
102-
expectedResolvedOutputs.put("outputName_2", "s3a://path/to/outputName_2");
103-
expectedResolvedOutputs.put("outputName_3", "s3a://path/to/outputName_3");
104-
100+
expectedResolvedOutputs.put("s3a://path/to/outputName_1", "outputName_1");
101+
expectedResolvedOutputs.put("s3a://path/to/outputName_2", "outputName_2");
102+
expectedResolvedOutputs.put("s3a://path/to/outputName_3", "outputName_3");
105103

106104
NuFacetBuilder builder = new NuFacetBuilder(openLineageContext);
107105
Map<String, RunFacet> runFacetMap = new HashMap<>();

0 commit comments

Comments
 (0)