| 44176 |
default
| collect at /opt/app/scheduler/jobs/dedup_job.py:386
+details
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Filter isnotnull(canonical_id#17308591)
+- InMemoryTableScan [brand_context_id#17308583, supplier_id#17308584, canonical_id#17308591, is_primary#17308592, entity_name#17308585, city#17308586, state#17308587, country#17308588, threshold#17308593, create_date#17308594], [isnotnull(canonical_id#17308591)]
+- InMemoryRelation [brand_context_id#17308583, supplier_id#17308584, entity_name#17308585, city#17308586, state#17308587, country#17308588, supplier_type#17308589, declaring_supplier_id#17308590, canonical_id#17308591, is_primary#17308592, threshold#17308593, create_date#17308594], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
...
org.apache.spark.sql.Dataset.collectToPython(Dataset.scala:4145)
jdk.internal.reflect.GeneratedMethodAccessor253.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
py4j.ClientServerConnection.run(ClientServerConnection.java:106)
java.base/java.lang.Thread.run(Unknown Source)
|
2026/10/20 02:12:55
| 0.1 s |
| 2.2 KiB | | | 80.0 B |