| Stage Id ▾ | Pool Name | Description | Submitted | Duration | Tasks: Succeeded/Total | Input | Output | Shuffle Read | Shuffle Write |
|---|---|---|---|---|---|---|---|---|---|
| 35031 | default | jdbc at <unknown>:0
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Filter isnotnull(canonical_id#17573030)
+- InMemoryTableScan [brand_context_id#17573022, supplier_id#17573023, canonical_id#17573030, is_primary#17573031, entity_name#17573024, city#17573025, state#17573026, country#17573027, threshold#17573032, create_date#17573033], [isnotnull(canonical_id#17573030)]
+- InMemoryRelation [brand_context_id#17573022, supplier_id#17573023, entity_name#17573024, city#17573025, state#17573026, country#17573027, supplier_type#17573028, declaring_supplier_id#17573029, canonical_id#17573030, is_primary#17573031, threshold#17573032, create_date#17573033], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
...
org.apache.spark.sql.DataFrameWriter.jdbc(DataFrameWriter.scala:756) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 10:04:05 | 0.1 s |
21/21
| 2.2 KiB | |||
| 35028 | default | count at <unknown>:0 org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 10:04:05 | 7 ms |
1/1
| 1179.0 B | |||
| 35024 | default | count at <unknown>:0
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Filter isnotnull(canonical_id#17573030)
+- InMemoryTableScan [brand_context_id#17573022, supplier_id#17573023, canonical_id#17573030, is_primary#17573031, entity_name#17573024, city#17573025, state#17573026, country#17573027, threshold#17573032, create_date#17573033], [isnotnull(canonical_id#17573030)]
+- InMemoryRelation [brand_context_id#17573022, supplier_id#17573023, entity_name#17573024, city#17573025, state#17573026, country#17573027, supplier_type#17573028, declaring_supplier_id#17573029, canonical_id#17573030, is_primary#17573031, threshold#17573032, create_date#17573033], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
...
org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 10:04:05 | 85 ms |
21/21
| 2.2 KiB | 1179.0 B | ||
| 35021 | default | collect at /opt/app/scheduler/jobs/dedup_job.py:386 org.apache.spark.sql.Dataset.collectToPython(Dataset.scala:4145) jdk.internal.reflect.GeneratedMethodAccessor272.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 10:04:05 | 10 ms |
1/1
| 80.0 B | |||
| 35017 | default | collect at /opt/app/scheduler/jobs/dedup_job.py:386
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Filter isnotnull(canonical_id#17573030)
+- InMemoryTableScan [brand_context_id#17573022, supplier_id#17573023, canonical_id#17573030, is_primary#17573031, entity_name#17573024, city#17573025, state#17573026, country#17573027, threshold#17573032, create_date#17573033], [isnotnull(canonical_id#17573030)]
+- InMemoryRelation [brand_context_id#17573022, supplier_id#17573023, entity_name#17573024, city#17573025, state#17573026, country#17573027, supplier_type#17573028, declaring_supplier_id#17573029, canonical_id#17573030, is_primary#17573031, threshold#17573032, create_date#17573033], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
...
org.apache.spark.sql.Dataset.collectToPython(Dataset.scala:4145) jdk.internal.reflect.GeneratedMethodAccessor272.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 10:04:05 | 0.1 s |
21/21
| 2.2 KiB | 80.0 B | ||
| 35014 | default | count at <unknown>:0 org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 10:04:05 | 5 ms |
1/1
| 1179.0 B | |||
| 35010 | default | count at <unknown>:0
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Filter isnotnull(canonical_id#17573030)
+- InMemoryTableScan [brand_context_id#17573022, supplier_id#17573023, canonical_id#17573030, is_primary#17573031, entity_name#17573024, city#17573025, state#17573026, country#17573027, threshold#17573032, create_date#17573033], [isnotnull(canonical_id#17573030)]
+- InMemoryRelation [brand_context_id#17573022, supplier_id#17573023, entity_name#17573024, city#17573025, state#17573026, country#17573027, supplier_type#17573028, declaring_supplier_id#17573029, canonical_id#17573030, is_primary#17573031, threshold#17573032, create_date#17573033], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
...
org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 10:04:04 | 83 ms |
21/21
| 2.2 KiB | 1179.0 B | ||
| 35007 | default | count at <unknown>:0
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Filter isnotnull(canonical_id#17573030)
+- InMemoryTableScan [brand_context_id#17573022, supplier_id#17573023, canonical_id#17573030, is_primary#17573031, entity_name#17573024, city#17573025, state#17573026, country#17573027, threshold#17573032, create_date#17573033], [isnotnull(canonical_id#17573030)]
+- InMemoryRelation [brand_context_id#17573022, supplier_id#17573023, entity_name#17573024, city#17573025, state#17573026, country#17573027, supplier_type#17573028, declaring_supplier_id#17573029, canonical_id#17573030, is_primary#17573031, threshold#17573032, create_date#17573033], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
...
org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 10:04:04 | 0.1 s |
21/21
| 1042.4 KiB | 175.0 B | ||
| 35004 | default | $anonfun$withThreadLocalCaptured$1 at <unknown>:0 org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 10:04:04 | 15 ms |
1/1
| ||||
| 35002 | default | count at <unknown>:0
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Filter isnotnull(canonical_id#17573030)
+- InMemoryTableScan [brand_context_id#17573022, supplier_id#17573023, canonical_id#17573030, is_primary#17573031, entity_name#17573024, city#17573025, state#17573026, country#17573027, threshold#17573032, create_date#17573033], [isnotnull(canonical_id#17573030)]
+- InMemoryRelation [brand_context_id#17573022, supplier_id#17573023, entity_name#17573024, city#17573025, state#17573026, country#17573027, supplier_type#17573028, declaring_supplier_id#17573029, canonical_id#17573030, is_primary#17573031, threshold#17573032, create_date#17573033], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
...
org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 10:04:04 | 38 ms |
1/1
| 1042.4 KiB | |||
| 35001 | default | count at <unknown>:0
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Filter isnotnull(canonical_id#17573030)
+- InMemoryTableScan [brand_context_id#17573022, supplier_id#17573023, canonical_id#17573030, is_primary#17573031, entity_name#17573024, city#17573025, state#17573026, country#17573027, threshold#17573032, create_date#17573033], [isnotnull(canonical_id#17573030)]
+- InMemoryRelation [brand_context_id#17573022, supplier_id#17573023, entity_name#17573024, city#17573025, state#17573026, country#17573027, supplier_type#17573028, declaring_supplier_id#17573029, canonical_id#17573030, is_primary#17573031, threshold#17573032, create_date#17573033], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
...
org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 10:04:04 | 12 ms |
1/1
| 1042.4 KiB | |||
| 35000 | default | $anonfun$withThreadLocalCaptured$1 at <unknown>:0
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Project [brand_context_id#17573022, supplier_id#17573023, entity_name#17573024, city#17573025, state#17573026, country#17573027, supplier_type#17573028, declaring_supplier_id#17573029, threshold#17573032, create_date#17573033]
+- Filter isnull(canonical_id#17573030)
+- InMemoryTableScan [brand_context_id#17573022, canonical_id#17573030, city#17573025, country#17573027, create_date#17573033, declaring_supplier_id#17573029, entity_name#17573024, state#17573026, supplier_id#17573023, supplier_type#17573028, threshold#17573032], [isnull(canonical_id#17573030)]
+- InMemoryRelation [brand_context_id#17573022, supplier_id#17573023, entity_name#17573024, city#17573025, state#17573026, country#17573027, supplier_type#17573028, declaring_supplier_id#17573029, canonical_id#17573030, is_primary#17573031, threshold#17573032, create_date#17573033], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SEL...
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 10:04:04 | 12 ms |
1/1
| 2040.0 B | |||
| 34999 | default | count at <unknown>:0
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Project [brand_context_id#17573022, supplier_id#17573023, entity_name#17573024, city#17573025, state#17573026, country#17573027, supplier_type#17573028, declaring_supplier_id#17573029, threshold#17573032, create_date#17573033]
+- Filter isnull(canonical_id#17573030)
+- InMemoryTableScan [brand_context_id#17573022, canonical_id#17573030, city#17573025, country#17573027, create_date#17573033, declaring_supplier_id#17573029, entity_name#17573024, state#17573026, supplier_id#17573023, supplier_type#17573028, threshold#17573032], [isnull(canonical_id#17573030)]
+- InMemoryRelation [brand_context_id#17573022, supplier_id#17573023, entity_name#17573024, city#17573025, state#17573026, country#17573027, supplier_type#17573028, declaring_supplier_id#17573029, canonical_id#17573030, is_primary#17573031, threshold#17573032, create_date#17573033], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SEL...
org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 10:04:04 | 16 ms |
1/1
| 2040.0 B | 175.0 B | ||
| 34998 | default | count at <unknown>:0
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Project [brand_context_id#17573022, supplier_id#17573023, entity_name#17573024, city#17573025, state#17573026, country#17573027, supplier_type#17573028, declaring_supplier_id#17573029, threshold#17573032, create_date#17573033]
+- Filter isnull(canonical_id#17573030)
+- InMemoryTableScan [brand_context_id#17573022, canonical_id#17573030, city#17573025, country#17573027, create_date#17573033, declaring_supplier_id#17573029, entity_name#17573024, state#17573026, supplier_id#17573023, supplier_type#17573028, threshold#17573032], [isnull(canonical_id#17573030)]
+- InMemoryRelation [brand_context_id#17573022, supplier_id#17573023, entity_name#17573024, city#17573025, state#17573026, country#17573027, supplier_type#17573028, declaring_supplier_id#17573029, canonical_id#17573030, is_primary#17573031, threshold#17573032, create_date#17573033], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SEL...
org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 10:04:04 | 0.1 s |
1/1
| 2040.0 B | |||
| 34997 | default | count at <unknown>:0
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Project [brand_context_id#17573022, supplier_id#17573023, entity_name#17573024, city#17573025, state#17573026, country#17573027, supplier_type#17573028, declaring_supplier_id#17573029, threshold#17573032, create_date#17573033]
+- Filter isnull(canonical_id#17573030)
+- InMemoryTableScan [brand_context_id#17573022, canonical_id#17573030, city#17573025, country#17573027, create_date#17573033, declaring_supplier_id#17573029, entity_name#17573024, state#17573026, supplier_id#17573023, supplier_type#17573028, threshold#17573032], [isnull(canonical_id#17573030)]
+- InMemoryRelation [brand_context_id#17573022, supplier_id#17573023, entity_name#17573024, city#17573025, state#17573026, country#17573027, supplier_type#17573028, declaring_supplier_id#17573029, canonical_id#17573030, is_primary#17573031, threshold#17573032, create_date#17573033], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SEL...
org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 10:04:04 | 0.2 s |
1/1
| 2040.0 B | |||
| 34996 | default | $anonfun$withThreadLocalCaptured$1 at <unknown>:0
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Project [brand_context_id#17573022, supplier_id#17573023, entity_name#17573024, city#17573025, state#17573026, country#17573027, supplier_type#17573028, declaring_supplier_id#17573029, threshold#17573032, create_date#17573033]
+- Filter isnull(canonical_id#17573030)
+- InMemoryTableScan [brand_context_id#17573022, canonical_id#17573030, city#17573025, country#17573027, create_date#17573033, declaring_supplier_id#17573029, entity_name#17573024, state#17573026, supplier_id#17573023, supplier_type#17573028, threshold#17573032], [isnull(canonical_id#17573030)]
+- InMemoryRelation [brand_context_id#17573022, supplier_id#17573023, entity_name#17573024, city#17573025, state#17573026, country#17573027, supplier_type#17573028, declaring_supplier_id#17573029, canonical_id#17573030, is_primary#17573031, threshold#17573032, create_date#17573033], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SEL...
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 10:04:04 | 47 ms |
1/1
| 2040.0 B | |||
| 34995 | default | $anonfun$withThreadLocalCaptured$1 at <unknown>:0
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Project [brand_context_id#17573022, supplier_id#17573023, entity_name#17573024, city#17573025, state#17573026, country#17573027, supplier_type#17573028, declaring_supplier_id#17573029, threshold#17573032, create_date#17573033]
+- Filter isnull(canonical_id#17573030)
+- InMemoryTableScan [brand_context_id#17573022, canonical_id#17573030, city#17573025, country#17573027, create_date#17573033, declaring_supplier_id#17573029, entity_name#17573024, state#17573026, supplier_id#17573023, supplier_type#17573028, threshold#17573032], [isnull(canonical_id#17573030)]
+- InMemoryRelation [brand_context_id#17573022, supplier_id#17573023, entity_name#17573024, city#17573025, state#17573026, country#17573027, supplier_type#17573028, declaring_supplier_id#17573029, canonical_id#17573030, is_primary#17573031, threshold#17573032, create_date#17573033], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SEL...
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 10:04:04 | 49 ms |
1/1
| 2040.0 B | |||
| 34994 | default | $anonfun$withThreadLocalCaptured$1 at <unknown>:0
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Project [brand_context_id#17573022, supplier_id#17573023, entity_name#17573024, city#17573025, state#17573026, country#17573027, supplier_type#17573028, declaring_supplier_id#17573029, threshold#17573032, create_date#17573033]
+- Filter isnull(canonical_id#17573030)
+- InMemoryTableScan [brand_context_id#17573022, canonical_id#17573030, city#17573025, country#17573027, create_date#17573033, declaring_supplier_id#17573029, entity_name#17573024, state#17573026, supplier_id#17573023, supplier_type#17573028, threshold#17573032], [isnull(canonical_id#17573030)]
+- InMemoryRelation [brand_context_id#17573022, supplier_id#17573023, entity_name#17573024, city#17573025, state#17573026, country#17573027, supplier_type#17573028, declaring_supplier_id#17573029, canonical_id#17573030, is_primary#17573031, threshold#17573032, create_date#17573033], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SEL...
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 10:04:04 | 41 ms |
1/1
| 2040.0 B | |||
| 34993 | default | count at <unknown>:0 org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 10:04:03 | 5 ms |
1/1
| 59.0 B | |||
| 34991 | default | count at <unknown>:0
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Filter isnotnull(canonical_id#17573030)
+- InMemoryTableScan [brand_context_id#17573022, supplier_id#17573023, canonical_id#17573030, is_primary#17573031, entity_name#17573024, city#17573025, state#17573026, country#17573027, threshold#17573032, create_date#17573033], [isnotnull(canonical_id#17573030)]
+- InMemoryRelation [brand_context_id#17573022, supplier_id#17573023, entity_name#17573024, city#17573025, state#17573026, country#17573027, supplier_type#17573028, declaring_supplier_id#17573029, canonical_id#17573030, is_primary#17573031, threshold#17573032, create_date#17573033], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
...
org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 10:04:03 | 6 ms |
1/1
| 1042.4 KiB | 59.0 B | ||
| 34990 | default | count at <unknown>:0 org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 10:04:03 | 5 ms |
1/1
| 59.0 B | |||
| 34988 | default | count at <unknown>:0
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Filter isnotnull(canonical_id#17573030)
+- InMemoryTableScan [brand_context_id#17573022, supplier_id#17573023, canonical_id#17573030, is_primary#17573031, entity_name#17573024, city#17573025, state#17573026, country#17573027, threshold#17573032, create_date#17573033], [isnotnull(canonical_id#17573030)]
+- InMemoryRelation [brand_context_id#17573022, supplier_id#17573023, entity_name#17573024, city#17573025, state#17573026, country#17573027, supplier_type#17573028, declaring_supplier_id#17573029, canonical_id#17573030, is_primary#17573031, threshold#17573032, create_date#17573033], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
...
org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 10:04:03 | 24 ms |
1/1
| 1042.4 KiB | 59.0 B | ||
| 34987 | default | count at <unknown>:0
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Filter isnotnull(canonical_id#17573030)
+- InMemoryTableScan [brand_context_id#17573022, supplier_id#17573023, canonical_id#17573030, is_primary#17573031, entity_name#17573024, city#17573025, state#17573026, country#17573027, threshold#17573032, create_date#17573033], [isnotnull(canonical_id#17573030)]
+- InMemoryRelation [brand_context_id#17573022, supplier_id#17573023, entity_name#17573024, city#17573025, state#17573026, country#17573027, supplier_type#17573028, declaring_supplier_id#17573029, canonical_id#17573030, is_primary#17573031, threshold#17573032, create_date#17573033], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
...
org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 10:04:03 | 84 ms |
1/1
| 1097.4 KiB | |||
| 34986 | default | count at <unknown>:0 org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 10:04:03 | 5 ms |
1/1
| 59.0 B | |||
| 34984 | default | count at <unknown>:0
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Project [brand_context_id#17573022, supplier_id#17573023, entity_name#17573024, city#17573025, state#17573026, country#17573027, supplier_type#17573028, declaring_supplier_id#17573029, threshold#17573032, create_date#17573033]
+- Filter isnull(canonical_id#17573030)
+- InMemoryTableScan [brand_context_id#17573022, canonical_id#17573030, city#17573025, country#17573027, create_date#17573033, declaring_supplier_id#17573029, entity_name#17573024, state#17573026, supplier_id#17573023, supplier_type#17573028, threshold#17573032], [isnull(canonical_id#17573030)]
+- InMemoryRelation [brand_context_id#17573022, supplier_id#17573023, entity_name#17573024, city#17573025, state#17573026, country#17573027, supplier_type#17573028, declaring_supplier_id#17573029, canonical_id#17573030, is_primary#17573031, threshold#17573032, create_date#17573033], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SEL...
org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 10:04:03 | 5 ms |
1/1
| 2040.0 B | 59.0 B | ||
| 34983 | default | count at <unknown>:0
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Project [brand_context_id#17573022, supplier_id#17573023, entity_name#17573024, city#17573025, state#17573026, country#17573027, supplier_type#17573028, declaring_supplier_id#17573029, threshold#17573032, create_date#17573033]
+- Filter isnull(canonical_id#17573030)
+- InMemoryTableScan [brand_context_id#17573022, canonical_id#17573030, city#17573025, country#17573027, create_date#17573033, declaring_supplier_id#17573029, entity_name#17573024, state#17573026, supplier_id#17573023, supplier_type#17573028, threshold#17573032], [isnull(canonical_id#17573030)]
+- InMemoryRelation [brand_context_id#17573022, supplier_id#17573023, entity_name#17573024, city#17573025, state#17573026, country#17573027, supplier_type#17573028, declaring_supplier_id#17573029, canonical_id#17573030, is_primary#17573031, threshold#17573032, create_date#17573033], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SEL...
org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 10:04:03 | 10 ms |
1/1
| 1097.4 KiB | |||
| 34982 | default | count at <unknown>:0 org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 10:04:03 | 5 ms |
1/1
| 59.0 B | |||
| 34980 | default | count at <unknown>:0 org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 10:04:03 | 6 ms |
1/1
| 1097.4 KiB | 59.0 B | ||
| 34979 | default | count at <unknown>:0 org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 10:04:03 | 0.4 s |
1/1
| ||||
| 34978 | default | jdbc at <unknown>:0
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Filter isnotnull(canonical_id#17566325)
+- InMemoryTableScan [brand_context_id#17566317, supplier_id#17566318, canonical_id#17566325, is_primary#17566326, entity_name#17566319, city#17566320, state#17566321, country#17566322, threshold#17566327, create_date#17566328], [isnotnull(canonical_id#17566325)]
+- InMemoryRelation [brand_context_id#17566317, supplier_id#17566318, entity_name#17566319, city#17566320, state#17566321, country#17566322, supplier_type#17566323, declaring_supplier_id#17566324, canonical_id#17566325, is_primary#17566326, threshold#17566327, create_date#17566328], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
...
org.apache.spark.sql.DataFrameWriter.jdbc(DataFrameWriter.scala:756) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:53:42 | 0.2 s |
21/21
| 4.1 KiB | |||
| 34975 | default | count at <unknown>:0 org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:53:41 | 6 ms |
1/1
| 1182.0 B | |||
| 34971 | default | count at <unknown>:0
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Filter isnotnull(canonical_id#17566325)
+- InMemoryTableScan [brand_context_id#17566317, supplier_id#17566318, canonical_id#17566325, is_primary#17566326, entity_name#17566319, city#17566320, state#17566321, country#17566322, threshold#17566327, create_date#17566328], [isnotnull(canonical_id#17566325)]
+- InMemoryRelation [brand_context_id#17566317, supplier_id#17566318, entity_name#17566319, city#17566320, state#17566321, country#17566322, supplier_type#17566323, declaring_supplier_id#17566324, canonical_id#17566325, is_primary#17566326, threshold#17566327, create_date#17566328], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
...
org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:53:41 | 96 ms |
21/21
| 4.1 KiB | 1182.0 B | ||
| 34968 | default | collect at /opt/app/scheduler/jobs/dedup_job.py:386 org.apache.spark.sql.Dataset.collectToPython(Dataset.scala:4145) jdk.internal.reflect.GeneratedMethodAccessor272.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:53:41 | 16 ms |
1/1
| 160.0 B | |||
| 34964 | default | collect at /opt/app/scheduler/jobs/dedup_job.py:386
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Filter isnotnull(canonical_id#17566325)
+- InMemoryTableScan [brand_context_id#17566317, supplier_id#17566318, canonical_id#17566325, is_primary#17566326, entity_name#17566319, city#17566320, state#17566321, country#17566322, threshold#17566327, create_date#17566328], [isnotnull(canonical_id#17566325)]
+- InMemoryRelation [brand_context_id#17566317, supplier_id#17566318, entity_name#17566319, city#17566320, state#17566321, country#17566322, supplier_type#17566323, declaring_supplier_id#17566324, canonical_id#17566325, is_primary#17566326, threshold#17566327, create_date#17566328], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
...
org.apache.spark.sql.Dataset.collectToPython(Dataset.scala:4145) jdk.internal.reflect.GeneratedMethodAccessor272.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:53:41 | 0.4 s |
21/21
| 4.1 KiB | 160.0 B | ||
| 34961 | default | count at <unknown>:0 org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:53:40 | 89 ms |
1/1
| 1182.0 B | |||
| 34957 | default | count at <unknown>:0
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Filter isnotnull(canonical_id#17566325)
+- InMemoryTableScan [brand_context_id#17566317, supplier_id#17566318, canonical_id#17566325, is_primary#17566326, entity_name#17566319, city#17566320, state#17566321, country#17566322, threshold#17566327, create_date#17566328], [isnotnull(canonical_id#17566325)]
+- InMemoryRelation [brand_context_id#17566317, supplier_id#17566318, entity_name#17566319, city#17566320, state#17566321, country#17566322, supplier_type#17566323, declaring_supplier_id#17566324, canonical_id#17566325, is_primary#17566326, threshold#17566327, create_date#17566328], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
...
org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:53:40 | 0.2 s |
21/21
| 4.1 KiB | 1182.0 B | ||
| 34954 | default | count at <unknown>:0
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Filter isnotnull(canonical_id#17566325)
+- InMemoryTableScan [brand_context_id#17566317, supplier_id#17566318, canonical_id#17566325, is_primary#17566326, entity_name#17566319, city#17566320, state#17566321, country#17566322, threshold#17566327, create_date#17566328], [isnotnull(canonical_id#17566325)]
+- InMemoryRelation [brand_context_id#17566317, supplier_id#17566318, entity_name#17566319, city#17566320, state#17566321, country#17566322, supplier_type#17566323, declaring_supplier_id#17566324, canonical_id#17566325, is_primary#17566326, threshold#17566327, create_date#17566328], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
...
org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:53:40 | 0.3 s |
21/21
| 1042.3 KiB | 581.0 B | ||
| 34951 | default | $anonfun$withThreadLocalCaptured$1 at <unknown>:0 org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:53:40 | 28 ms |
1/1
| 151.0 B | |||
| 34949 | default | count at <unknown>:0
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Filter isnotnull(canonical_id#17566325)
+- InMemoryTableScan [brand_context_id#17566317, supplier_id#17566318, canonical_id#17566325, is_primary#17566326, entity_name#17566319, city#17566320, state#17566321, country#17566322, threshold#17566327, create_date#17566328], [isnotnull(canonical_id#17566325)]
+- InMemoryRelation [brand_context_id#17566317, supplier_id#17566318, entity_name#17566319, city#17566320, state#17566321, country#17566322, supplier_type#17566323, declaring_supplier_id#17566324, canonical_id#17566325, is_primary#17566326, threshold#17566327, create_date#17566328], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
...
org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:53:36 | 2 s |
1/1
| 1042.3 KiB | 151.0 B | ||
| 34948 | default | count at <unknown>:0
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Filter isnotnull(canonical_id#17566325)
+- InMemoryTableScan [brand_context_id#17566317, supplier_id#17566318, canonical_id#17566325, is_primary#17566326, entity_name#17566319, city#17566320, state#17566321, country#17566322, threshold#17566327, create_date#17566328], [isnotnull(canonical_id#17566325)]
+- InMemoryRelation [brand_context_id#17566317, supplier_id#17566318, entity_name#17566319, city#17566320, state#17566321, country#17566322, supplier_type#17566323, declaring_supplier_id#17566324, canonical_id#17566325, is_primary#17566326, threshold#17566327, create_date#17566328], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
...
org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:53:36 | 2 s |
1/1
| 1042.3 KiB | 239.0 B | ||
| 34947 | default | $anonfun$withThreadLocalCaptured$1 at <unknown>:0 org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:53:35 | 11 ms |
1/1
| 362.0 B | |||
| 34945 | default | $anonfun$withThreadLocalCaptured$1 at <unknown>:0 org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:53:35 | 8 ms |
1/1
| 464.0 B | |||
| 34943 | default | count at <unknown>:0
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Filter isnotnull(canonical_id#17566325)
+- InMemoryTableScan [brand_context_id#17566317, supplier_id#17566318, canonical_id#17566325, is_primary#17566326, entity_name#17566319, city#17566320, state#17566321, country#17566322, threshold#17566327, create_date#17566328], [isnotnull(canonical_id#17566325)]
+- InMemoryRelation [brand_context_id#17566317, supplier_id#17566318, entity_name#17566319, city#17566320, state#17566321, country#17566322, supplier_type#17566323, declaring_supplier_id#17566324, canonical_id#17566325, is_primary#17566326, threshold#17566327, create_date#17566328], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
...
org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:53:35 | 55 ms |
1/1
| 1042.3 KiB | |||
| 34942 | default | count at <unknown>:0
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Filter isnotnull(canonical_id#17566325)
+- InMemoryTableScan [brand_context_id#17566317, supplier_id#17566318, canonical_id#17566325, is_primary#17566326, entity_name#17566319, city#17566320, state#17566321, country#17566322, threshold#17566327, create_date#17566328], [isnotnull(canonical_id#17566325)]
+- InMemoryRelation [brand_context_id#17566317, supplier_id#17566318, entity_name#17566319, city#17566320, state#17566321, country#17566322, supplier_type#17566323, declaring_supplier_id#17566324, canonical_id#17566325, is_primary#17566326, threshold#17566327, create_date#17566328], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
...
org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:53:35 | 20 ms |
1/1
| 1042.3 KiB | |||
| 34941 | default | count at <unknown>:0
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Filter isnotnull(canonical_id#17566325)
+- InMemoryTableScan [brand_context_id#17566317, supplier_id#17566318, canonical_id#17566325, is_primary#17566326, entity_name#17566319, city#17566320, state#17566321, country#17566322, threshold#17566327, create_date#17566328], [isnotnull(canonical_id#17566325)]
+- InMemoryRelation [brand_context_id#17566317, supplier_id#17566318, entity_name#17566319, city#17566320, state#17566321, country#17566322, supplier_type#17566323, declaring_supplier_id#17566324, canonical_id#17566325, is_primary#17566326, threshold#17566327, create_date#17566328], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
...
org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:53:35 | 27 ms |
1/1
| 1042.3 KiB | |||
| 34940 | default | $anonfun$withThreadLocalCaptured$1 at <unknown>:0
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Project [brand_context_id#17566317, supplier_id#17566318, entity_name#17566319, city#17566320, state#17566321, country#17566322, supplier_type#17566323, declaring_supplier_id#17566324, threshold#17566327, create_date#17566328]
+- Filter isnull(canonical_id#17566325)
+- InMemoryTableScan [brand_context_id#17566317, canonical_id#17566325, city#17566320, country#17566322, create_date#17566328, declaring_supplier_id#17566324, entity_name#17566319, state#17566321, supplier_id#17566318, supplier_type#17566323, threshold#17566327], [isnull(canonical_id#17566325)]
+- InMemoryRelation [brand_context_id#17566317, supplier_id#17566318, entity_name#17566319, city#17566320, state#17566321, country#17566322, supplier_type#17566323, declaring_supplier_id#17566324, canonical_id#17566325, is_primary#17566326, threshold#17566327, create_date#17566328], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SEL...
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:53:35 | 60 ms |
1/1
| 2.2 KiB | |||
| 34939 | default | count at <unknown>:0
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Project [brand_context_id#17566317, supplier_id#17566318, entity_name#17566319, city#17566320, state#17566321, country#17566322, supplier_type#17566323, declaring_supplier_id#17566324, threshold#17566327, create_date#17566328]
+- Filter isnull(canonical_id#17566325)
+- InMemoryTableScan [brand_context_id#17566317, canonical_id#17566325, city#17566320, country#17566322, create_date#17566328, declaring_supplier_id#17566324, entity_name#17566319, state#17566321, supplier_id#17566318, supplier_type#17566323, threshold#17566327], [isnull(canonical_id#17566325)]
+- InMemoryRelation [brand_context_id#17566317, supplier_id#17566318, entity_name#17566319, city#17566320, state#17566321, country#17566322, supplier_type#17566323, declaring_supplier_id#17566324, canonical_id#17566325, is_primary#17566326, threshold#17566327, create_date#17566328], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SEL...
org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:53:35 | 0.2 s |
1/1
| 2.2 KiB | 362.0 B | ||
| 34938 | default | count at <unknown>:0
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Project [brand_context_id#17566317, supplier_id#17566318, entity_name#17566319, city#17566320, state#17566321, country#17566322, supplier_type#17566323, declaring_supplier_id#17566324, threshold#17566327, create_date#17566328]
+- Filter isnull(canonical_id#17566325)
+- InMemoryTableScan [brand_context_id#17566317, canonical_id#17566325, city#17566320, country#17566322, create_date#17566328, declaring_supplier_id#17566324, entity_name#17566319, state#17566321, supplier_id#17566318, supplier_type#17566323, threshold#17566327], [isnull(canonical_id#17566325)]
+- InMemoryRelation [brand_context_id#17566317, supplier_id#17566318, entity_name#17566319, city#17566320, state#17566321, country#17566322, supplier_type#17566323, declaring_supplier_id#17566324, canonical_id#17566325, is_primary#17566326, threshold#17566327, create_date#17566328], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SEL...
org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:53:35 | 7 ms |
1/1
| 2.2 KiB | 342.0 B | ||
| 34937 | default | count at <unknown>:0
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Project [brand_context_id#17566317, supplier_id#17566318, entity_name#17566319, city#17566320, state#17566321, country#17566322, supplier_type#17566323, declaring_supplier_id#17566324, threshold#17566327, create_date#17566328]
+- Filter isnull(canonical_id#17566325)
+- InMemoryTableScan [brand_context_id#17566317, canonical_id#17566325, city#17566320, country#17566322, create_date#17566328, declaring_supplier_id#17566324, entity_name#17566319, state#17566321, supplier_id#17566318, supplier_type#17566323, threshold#17566327], [isnull(canonical_id#17566325)]
+- InMemoryRelation [brand_context_id#17566317, supplier_id#17566318, entity_name#17566319, city#17566320, state#17566321, country#17566322, supplier_type#17566323, declaring_supplier_id#17566324, canonical_id#17566325, is_primary#17566326, threshold#17566327, create_date#17566328], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SEL...
org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:53:35 | 0.2 s |
1/1
| 2.2 KiB | 464.0 B | ||
| 34936 | default | $anonfun$withThreadLocalCaptured$1 at <unknown>:0
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Project [brand_context_id#17566317, supplier_id#17566318, entity_name#17566319, city#17566320, state#17566321, country#17566322, supplier_type#17566323, declaring_supplier_id#17566324, threshold#17566327, create_date#17566328]
+- Filter isnull(canonical_id#17566325)
+- InMemoryTableScan [brand_context_id#17566317, canonical_id#17566325, city#17566320, country#17566322, create_date#17566328, declaring_supplier_id#17566324, entity_name#17566319, state#17566321, supplier_id#17566318, supplier_type#17566323, threshold#17566327], [isnull(canonical_id#17566325)]
+- InMemoryRelation [brand_context_id#17566317, supplier_id#17566318, entity_name#17566319, city#17566320, state#17566321, country#17566322, supplier_type#17566323, declaring_supplier_id#17566324, canonical_id#17566325, is_primary#17566326, threshold#17566327, create_date#17566328], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SEL...
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:53:35 | 14 ms |
1/1
| 2.2 KiB | |||
| 34935 | default | $anonfun$withThreadLocalCaptured$1 at <unknown>:0
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Project [brand_context_id#17566317, supplier_id#17566318, entity_name#17566319, city#17566320, state#17566321, country#17566322, supplier_type#17566323, declaring_supplier_id#17566324, threshold#17566327, create_date#17566328]
+- Filter isnull(canonical_id#17566325)
+- InMemoryTableScan [brand_context_id#17566317, canonical_id#17566325, city#17566320, country#17566322, create_date#17566328, declaring_supplier_id#17566324, entity_name#17566319, state#17566321, supplier_id#17566318, supplier_type#17566323, threshold#17566327], [isnull(canonical_id#17566325)]
+- InMemoryRelation [brand_context_id#17566317, supplier_id#17566318, entity_name#17566319, city#17566320, state#17566321, country#17566322, supplier_type#17566323, declaring_supplier_id#17566324, canonical_id#17566325, is_primary#17566326, threshold#17566327, create_date#17566328], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SEL...
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:53:35 | 80 ms |
1/1
| 2.2 KiB | |||
| 34934 | default | $anonfun$withThreadLocalCaptured$1 at <unknown>:0
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Project [brand_context_id#17566317, supplier_id#17566318, entity_name#17566319, city#17566320, state#17566321, country#17566322, supplier_type#17566323, declaring_supplier_id#17566324, threshold#17566327, create_date#17566328]
+- Filter isnull(canonical_id#17566325)
+- InMemoryTableScan [brand_context_id#17566317, canonical_id#17566325, city#17566320, country#17566322, create_date#17566328, declaring_supplier_id#17566324, entity_name#17566319, state#17566321, supplier_id#17566318, supplier_type#17566323, threshold#17566327], [isnull(canonical_id#17566325)]
+- InMemoryRelation [brand_context_id#17566317, supplier_id#17566318, entity_name#17566319, city#17566320, state#17566321, country#17566322, supplier_type#17566323, declaring_supplier_id#17566324, canonical_id#17566325, is_primary#17566326, threshold#17566327, create_date#17566328], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SEL...
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:53:35 | 17 ms |
1/1
| 2.2 KiB | |||
| 34933 | default | count at <unknown>:0 org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:53:34 | 6 ms |
1/1
| 59.0 B | |||
| 34931 | default | count at <unknown>:0
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Filter isnotnull(canonical_id#17566325)
+- InMemoryTableScan [brand_context_id#17566317, supplier_id#17566318, canonical_id#17566325, is_primary#17566326, entity_name#17566319, city#17566320, state#17566321, country#17566322, threshold#17566327, create_date#17566328], [isnotnull(canonical_id#17566325)]
+- InMemoryRelation [brand_context_id#17566317, supplier_id#17566318, entity_name#17566319, city#17566320, state#17566321, country#17566322, supplier_type#17566323, declaring_supplier_id#17566324, canonical_id#17566325, is_primary#17566326, threshold#17566327, create_date#17566328], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
...
org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:53:34 | 25 ms |
1/1
| 1042.3 KiB | 59.0 B | ||
| 34930 | default | count at <unknown>:0 org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:53:34 | 6 ms |
1/1
| 59.0 B | |||
| 34928 | default | count at <unknown>:0
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Filter isnotnull(canonical_id#17566325)
+- InMemoryTableScan [brand_context_id#17566317, supplier_id#17566318, canonical_id#17566325, is_primary#17566326, entity_name#17566319, city#17566320, state#17566321, country#17566322, threshold#17566327, create_date#17566328], [isnotnull(canonical_id#17566325)]
+- InMemoryRelation [brand_context_id#17566317, supplier_id#17566318, entity_name#17566319, city#17566320, state#17566321, country#17566322, supplier_type#17566323, declaring_supplier_id#17566324, canonical_id#17566325, is_primary#17566326, threshold#17566327, create_date#17566328], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
...
org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:53:34 | 7 ms |
1/1
| 1042.3 KiB | 59.0 B | ||
| 34927 | default | count at <unknown>:0
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Filter isnotnull(canonical_id#17566325)
+- InMemoryTableScan [brand_context_id#17566317, supplier_id#17566318, canonical_id#17566325, is_primary#17566326, entity_name#17566319, city#17566320, state#17566321, country#17566322, threshold#17566327, create_date#17566328], [isnotnull(canonical_id#17566325)]
+- InMemoryRelation [brand_context_id#17566317, supplier_id#17566318, entity_name#17566319, city#17566320, state#17566321, country#17566322, supplier_type#17566323, declaring_supplier_id#17566324, canonical_id#17566325, is_primary#17566326, threshold#17566327, create_date#17566328], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
...
org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:53:34 | 0.1 s |
1/1
| 1097.3 KiB | |||
| 34926 | default | count at <unknown>:0 org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:53:34 | 5 ms |
1/1
| 59.0 B | |||
| 34924 | default | count at <unknown>:0
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Project [brand_context_id#17566317, supplier_id#17566318, entity_name#17566319, city#17566320, state#17566321, country#17566322, supplier_type#17566323, declaring_supplier_id#17566324, threshold#17566327, create_date#17566328]
+- Filter isnull(canonical_id#17566325)
+- InMemoryTableScan [brand_context_id#17566317, canonical_id#17566325, city#17566320, country#17566322, create_date#17566328, declaring_supplier_id#17566324, entity_name#17566319, state#17566321, supplier_id#17566318, supplier_type#17566323, threshold#17566327], [isnull(canonical_id#17566325)]
+- InMemoryRelation [brand_context_id#17566317, supplier_id#17566318, entity_name#17566319, city#17566320, state#17566321, country#17566322, supplier_type#17566323, declaring_supplier_id#17566324, canonical_id#17566325, is_primary#17566326, threshold#17566327, create_date#17566328], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SEL...
org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:53:34 | 6 ms |
1/1
| 2.2 KiB | 59.0 B | ||
| 34923 | default | count at <unknown>:0
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Project [brand_context_id#17566317, supplier_id#17566318, entity_name#17566319, city#17566320, state#17566321, country#17566322, supplier_type#17566323, declaring_supplier_id#17566324, threshold#17566327, create_date#17566328]
+- Filter isnull(canonical_id#17566325)
+- InMemoryTableScan [brand_context_id#17566317, canonical_id#17566325, city#17566320, country#17566322, create_date#17566328, declaring_supplier_id#17566324, entity_name#17566319, state#17566321, supplier_id#17566318, supplier_type#17566323, threshold#17566327], [isnull(canonical_id#17566325)]
+- InMemoryRelation [brand_context_id#17566317, supplier_id#17566318, entity_name#17566319, city#17566320, state#17566321, country#17566322, supplier_type#17566323, declaring_supplier_id#17566324, canonical_id#17566325, is_primary#17566326, threshold#17566327, create_date#17566328], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SEL...
org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:53:34 | 25 ms |
1/1
| 1097.3 KiB | |||
| 34922 | default | count at <unknown>:0 org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:53:34 | 44 ms |
1/1
| 59.0 B | |||
| 34920 | default | count at <unknown>:0 org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:53:34 | 14 ms |
1/1
| 1097.3 KiB | 59.0 B | ||
| 34919 | default | count at <unknown>:0 org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:53:33 | 0.4 s |
1/1
| ||||
| 34918 | default | collect at /opt/app/scheduler/jobs/memory_probe.py:76 org.apache.spark.rdd.RDD.collect(RDD.scala:1048) org.apache.spark.api.python.PythonRDD$.collectAndServe(PythonRDD.scala:195) org.apache.spark.api.python.PythonRDD.collectAndServe(PythonRDD.scala) jdk.internal.reflect.GeneratedMethodAccessor267.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:52:54 | 0.2 s |
2/2
| ||||
| 34917 | MemgraphDeclaredBatch-6aacf5861a41b434b2a90520_1 | foreachPartition at /opt/app/scheduler/jobs/memgraph_loader_job.py:2107 org.apache.spark.rdd.RDD.collect(RDD.scala:1048) org.apache.spark.api.python.PythonRDD$.collectAndServe(PythonRDD.scala:195) org.apache.spark.api.python.PythonRDD.collectAndServe(PythonRDD.scala) jdk.internal.reflect.GeneratedMethodAccessor267.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:52:52 | 0.9 s |
1/1
| 12.9 KiB | 5.0 KiB | ||
| 34910 | MemgraphDeclaredBatch-6aacf5861a41b434b2a90520_1 | javaToPython at <unknown>:0 org.apache.spark.sql.Dataset.javaToPython(Dataset.scala:4139) jdk.internal.reflect.GeneratedMethodAccessor228.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:52:52 | 0.3 s |
22/22
| 12.9 KiB | 4.6 KiB | ||
| 34909 | MemgraphDeclaredBatch-6aacf5861a41b434b2a90520_1 | javaToPython at <unknown>:0 org.apache.spark.sql.Dataset.javaToPython(Dataset.scala:4139) jdk.internal.reflect.GeneratedMethodAccessor228.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:52:52 | 0.3 s |
22/22
| 12.9 KiB | 386.0 B | ||
| 34905 | MemgraphDeclaredBatch-6aacf5861a41b434b2a90520_1 | $anonfun$withThreadLocalCaptured$1 at <unknown>:0 org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:52:51 | 46 ms |
1/1
| 9.6 KiB | |||
| 34904 | MemgraphDeclaredBatch-6aacf5861a41b434b2a90520_1 | $anonfun$withThreadLocalCaptured$1 at <unknown>:0 org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:52:51 | 36 ms |
2/2
| 19.3 KiB | |||
| 34903 | MemgraphDeclaredBatch-6aacf5861a41b434b2a90520_1 | $anonfun$withThreadLocalCaptured$1 at <unknown>:0 org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:52:51 | 41 ms |
2/2
| 19.3 KiB | |||
| 34902 | MemgraphDeclaredBatch-6aacf5861a41b434b2a90520_1 | $anonfun$withThreadLocalCaptured$1 at <unknown>:0 org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:52:51 | 0.1 s |
1/1
| 9.6 KiB | |||
| 34901 | MemgraphDeclaredBatch-6aacf5861a41b434b2a90520_1 | $anonfun$withThreadLocalCaptured$1 at <unknown>:0 org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:52:51 | 53 ms |
1/1
| 9.6 KiB | |||
| 34899 | MemgraphDeclaredBatch-6aacf5861a41b434b2a90520_1 | foreachPartition at /opt/app/scheduler/jobs/memgraph_loader_job.py:2107 org.apache.spark.rdd.RDD.collect(RDD.scala:1048) org.apache.spark.api.python.PythonRDD$.collectAndServe(PythonRDD.scala:195) org.apache.spark.api.python.PythonRDD.collectAndServe(PythonRDD.scala) jdk.internal.reflect.GeneratedMethodAccessor267.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:52:47 | 0.8 s |
1/1
| 12.9 KiB | |||
| 34894 | MemgraphDeclaredBatch-6aacf5861a41b434b2a90520_1 | javaToPython at <unknown>:0 org.apache.spark.sql.Dataset.javaToPython(Dataset.scala:4139) jdk.internal.reflect.GeneratedMethodAccessor228.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:52:47 | 0.5 s |
20/20
| 1712.0 B | |||
| 34889 | MemgraphDeclaredBatch-6aacf5861a41b434b2a90520_1 | javaToPython at <unknown>:0 org.apache.spark.sql.Dataset.javaToPython(Dataset.scala:4139) jdk.internal.reflect.GeneratedMethodAccessor228.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:52:46 | 0.1 s |
1/1
| 234.0 B | 170.0 B | ||
| 34887 | MemgraphDeclaredBatch-6aacf5861a41b434b2a90520_1 | javaToPython at <unknown>:0 org.apache.spark.sql.Dataset.javaToPython(Dataset.scala:4139) jdk.internal.reflect.GeneratedMethodAccessor228.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:52:46 | 89 ms |
4/4
| 1248.0 B | |||
| 34886 | MemgraphDeclaredBatch-6aacf5861a41b434b2a90520_1 | javaToPython at <unknown>:0 org.apache.spark.sql.Dataset.javaToPython(Dataset.scala:4139) jdk.internal.reflect.GeneratedMethodAccessor228.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:52:46 | 0.2 s |
7/7
| 2.8 KiB | |||
| 34885 | MemgraphDeclaredBatch-6aacf5861a41b434b2a90520_1 | javaToPython at <unknown>:0 org.apache.spark.sql.Dataset.javaToPython(Dataset.scala:4139) jdk.internal.reflect.GeneratedMethodAccessor228.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:52:46 | 0.3 s |
11/11
| 4.0 KiB | 1542.0 B | ||
| 34884 | MemgraphDeclaredBatch-6aacf5861a41b434b2a90520_1 | javaToPython at <unknown>:0 org.apache.spark.sql.Dataset.javaToPython(Dataset.scala:4139) jdk.internal.reflect.GeneratedMethodAccessor228.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:52:45 | 0.2 s |
3/3
| 1592.0 B | 234.0 B | ||
| 34883 | MemgraphDeclaredBatch-6aacf5861a41b434b2a90520_1 | javaToPython at <unknown>:0 org.apache.spark.sql.Dataset.javaToPython(Dataset.scala:4139) jdk.internal.reflect.GeneratedMethodAccessor228.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:52:45 | 53 ms |
4/4
| 1248.0 B | |||
| 34882 | MemgraphDeclaredBatch-6aacf5861a41b434b2a90520_1 | javaToPython at <unknown>:0 org.apache.spark.sql.Dataset.javaToPython(Dataset.scala:4139) jdk.internal.reflect.GeneratedMethodAccessor228.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:52:45 | 39 ms |
4/4
| 1248.0 B | |||
| 34881 | MemgraphDeclaredBatch-6aacf5861a41b434b2a90520_1 | javaToPython at <unknown>:0 org.apache.spark.sql.Dataset.javaToPython(Dataset.scala:4139) jdk.internal.reflect.GeneratedMethodAccessor228.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:52:45 | 36 ms |
4/4
| 1248.0 B | |||
| 34880 | MemgraphDeclaredBatch-6aacf5861a41b434b2a90520_1 | javaToPython at <unknown>:0 org.apache.spark.sql.Dataset.javaToPython(Dataset.scala:4139) jdk.internal.reflect.GeneratedMethodAccessor228.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:52:45 | 0.4 s |
4/4
| 3.0 KiB | |||
| 34878 | MemgraphDeclaredBatch-6aacf5861a41b434b2a90520_1 | $anonfun$withThreadLocalCaptured$1 at <unknown>:0 org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:52:45 | 74 ms |
1/1
| 1072.0 B | |||
| 34877 | MemgraphDeclaredBatch-6aacf5861a41b434b2a90520_1 | javaToPython at <unknown>:0 org.apache.spark.sql.Dataset.javaToPython(Dataset.scala:4139) jdk.internal.reflect.GeneratedMethodAccessor228.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:52:45 | 0.3 s |
3/3
| 3.3 KiB | |||
| 34875 | MemgraphDeclaredBatch-6aacf5861a41b434b2a90520_1 | $anonfun$withThreadLocalCaptured$1 at <unknown>:0 org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:52:45 | 26 ms |
1/1
| 1072.0 B | |||
| 34873 | MemgraphDeclaredBatch-6aacf5861a41b434b2a90520_1 | foreachPartition at /opt/app/scheduler/jobs/memgraph_loader_job.py:2107 org.apache.spark.rdd.RDD.collect(RDD.scala:1048) org.apache.spark.api.python.PythonRDD$.collectAndServe(PythonRDD.scala:195) org.apache.spark.api.python.PythonRDD.collectAndServe(PythonRDD.scala) jdk.internal.reflect.GeneratedMethodAccessor267.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:52:43 | 0.8 s |
1/1
| 3.3 KiB | |||
| 34871 | MemgraphDeclaredBatch-6aacf5861a41b434b2a90520_1 | javaToPython at <unknown>:0 org.apache.spark.sql.Dataset.javaToPython(Dataset.scala:4139) jdk.internal.reflect.GeneratedMethodAccessor228.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:52:43 | 69 ms |
1/1
| 2.2 KiB | |||
| 34869 | MemgraphDeclaredBatch-6aacf5861a41b434b2a90520_1 | $anonfun$withThreadLocalCaptured$1 at <unknown>:0 org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:52:43 | 8 ms |
1/1
| 18.1 KiB | |||
| 34868 | MemgraphDeclaredBatch-6aacf5861a41b434b2a90520_1 | foreachPartition at /opt/app/scheduler/jobs/memgraph_loader_job.py:2107 org.apache.spark.rdd.RDD.collect(RDD.scala:1048) org.apache.spark.api.python.PythonRDD$.collectAndServe(PythonRDD.scala:195) org.apache.spark.api.python.PythonRDD.collectAndServe(PythonRDD.scala) jdk.internal.reflect.GeneratedMethodAccessor267.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:52:42 | 0.7 s |
1/1
| 3.0 KiB | |||
| 34866 | MemgraphDeclaredBatch-6aacf5861a41b434b2a90520_1 | javaToPython at <unknown>:0 org.apache.spark.sql.Dataset.javaToPython(Dataset.scala:4139) jdk.internal.reflect.GeneratedMethodAccessor228.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:52:42 | 34 ms |
2/2
| 2.3 KiB | |||
| 34864 | MemgraphDeclaredBatch-6aacf5861a41b434b2a90520_1 | $anonfun$withThreadLocalCaptured$1 at <unknown>:0 org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:52:42 | 13 ms |
1/1
| 18.1 KiB | |||
| 34863 | MemgraphDeclaredBatch-6aacf5861a41b434b2a90520_1 | foreachPartition at /opt/app/scheduler/jobs/memgraph_loader_job.py:2107 org.apache.spark.rdd.RDD.collect(RDD.scala:1048) org.apache.spark.api.python.PythonRDD$.collectAndServe(PythonRDD.scala:195) org.apache.spark.api.python.PythonRDD.collectAndServe(PythonRDD.scala) jdk.internal.reflect.GeneratedMethodAccessor267.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:52:41 | 0.7 s |
1/1
| 1072.0 B | |||
| 34861 | MemgraphDeclaredBatch-6aacf5861a41b434b2a90520_1 | javaToPython at <unknown>:0 org.apache.spark.sql.Dataset.javaToPython(Dataset.scala:4139) jdk.internal.reflect.GeneratedMethodAccessor228.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:52:40 | 0.8 s |
1/1
| 9.6 KiB | |||
| 34859 | MemgraphDeclaredBatch-6aacf5861a41b434b2a90520_1 | javaToPython at <unknown>:0 org.apache.spark.sql.Dataset.javaToPython(Dataset.scala:4139) jdk.internal.reflect.GeneratedMethodAccessor228.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:52:40 | 0.2 s |
1/1
| 9.6 KiB | |||
| 34857 | MemgraphDeclaredBatch-6aacf5861a41b434b2a90520_1 | $anonfun$withThreadLocalCaptured$1 at <unknown>:0 org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:52:40 | 6 ms |
1/1
| 408.0 B | |||
| 34856 | MemgraphDeclaredBatch-6aacf5861a41b434b2a90520_1 | $anonfun$withThreadLocalCaptured$1 at <unknown>:0 org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:52:40 | 8 ms |
1/1
| 656.0 B | |||
| 34855 | MemgraphDeclaredBatch-6aacf5861a41b434b2a90520_1 | $anonfun$withThreadLocalCaptured$1 at <unknown>:0 org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:52:40 | 10 ms |
1/1
| 2.3 KiB | |||
| 34853 | MemgraphDeclaredBatch-6aacf5861a41b434b2a90520_1 | foreachPartition at /opt/app/scheduler/jobs/memgraph_loader_job.py:2107 org.apache.spark.rdd.RDD.collect(RDD.scala:1048) org.apache.spark.api.python.PythonRDD$.collectAndServe(PythonRDD.scala:195) org.apache.spark.api.python.PythonRDD.collectAndServe(PythonRDD.scala) jdk.internal.reflect.GeneratedMethodAccessor267.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:52:39 | 0.7 s |
1/1
| 2.2 KiB | |||
| 34851 | MemgraphDeclaredBatch-6aacf5861a41b434b2a90520_1 | javaToPython at <unknown>:0 org.apache.spark.sql.Dataset.javaToPython(Dataset.scala:4139) jdk.internal.reflect.GeneratedMethodAccessor228.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | 2026/10/16 09:52:39 | 0.4 s |
1/1
| 9.6 KiB |
| Stage Id ▾ | Pool Name | Description | Submitted | Duration | Tasks: Succeeded/Total | Input | Output | Shuffle Read | Shuffle Write |
|---|---|---|---|---|---|---|---|---|---|
| 35030 | default | count at <unknown>:0 org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | Unknown | Unknown |
0/0
| ||||
| 35029 | default | count at <unknown>:0
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Project [brand_context_id#17573022, supplier_id#17573023, entity_name#17573024, city#17573025, state#17573026, country#17573027, supplier_type#17573028, declaring_supplier_id#17573029, threshold#17573032, create_date#17573033]
+- Filter isnull(canonical_id#17573030)
+- InMemoryTableScan [brand_context_id#17573022, canonical_id#17573030, city#17573025, country#17573027, create_date#17573033, declaring_supplier_id#17573029, entity_name#17573024, state#17573026, supplier_id#17573023, supplier_type#17573028, threshold#17573032], [isnull(canonical_id#17573030)]
+- InMemoryRelation [brand_context_id#17573022, supplier_id#17573023, entity_name#17573024, city#17573025, state#17573026, country#17573027, supplier_type#17573028, declaring_supplier_id#17573029, canonical_id#17573030, is_primary#17573031, threshold#17573032, create_date#17573033], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SEL...
org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | Unknown | Unknown |
0/1
| ||||
| 35027 | default | count at <unknown>:0
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Filter isnotnull(canonical_id#17573030)
+- InMemoryTableScan [brand_context_id#17573022, supplier_id#17573023, canonical_id#17573030, is_primary#17573031, entity_name#17573024, city#17573025, state#17573026, country#17573027, threshold#17573032, create_date#17573033], [isnotnull(canonical_id#17573030)]
+- InMemoryRelation [brand_context_id#17573022, supplier_id#17573023, entity_name#17573024, city#17573025, state#17573026, country#17573027, supplier_type#17573028, declaring_supplier_id#17573029, canonical_id#17573030, is_primary#17573031, threshold#17573032, create_date#17573033], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
...
org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | Unknown | Unknown |
0/21
| ||||
| 35026 | default | count at <unknown>:0
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Project [brand_context_id#17573022, supplier_id#17573023, entity_name#17573024, city#17573025, state#17573026, country#17573027, supplier_type#17573028, declaring_supplier_id#17573029, threshold#17573032, create_date#17573033]
+- Filter isnull(canonical_id#17573030)
+- InMemoryTableScan [brand_context_id#17573022, canonical_id#17573030, city#17573025, country#17573027, create_date#17573033, declaring_supplier_id#17573029, entity_name#17573024, state#17573026, supplier_id#17573023, supplier_type#17573028, threshold#17573032], [isnull(canonical_id#17573030)]
+- InMemoryRelation [brand_context_id#17573022, supplier_id#17573023, entity_name#17573024, city#17573025, state#17573026, country#17573027, supplier_type#17573028, declaring_supplier_id#17573029, canonical_id#17573030, is_primary#17573031, threshold#17573032, create_date#17573033], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SEL...
org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | Unknown | Unknown |
0/1
| ||||
| 35025 | default | count at <unknown>:0 org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | Unknown | Unknown |
0/0
| ||||
| 35023 | default | count at <unknown>:0
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Project [brand_context_id#17573022, supplier_id#17573023, entity_name#17573024, city#17573025, state#17573026, country#17573027, supplier_type#17573028, declaring_supplier_id#17573029, threshold#17573032, create_date#17573033]
+- Filter isnull(canonical_id#17573030)
+- InMemoryTableScan [brand_context_id#17573022, canonical_id#17573030, city#17573025, country#17573027, create_date#17573033, declaring_supplier_id#17573029, entity_name#17573024, state#17573026, supplier_id#17573023, supplier_type#17573028, threshold#17573032], [isnull(canonical_id#17573030)]
+- InMemoryRelation [brand_context_id#17573022, supplier_id#17573023, entity_name#17573024, city#17573025, state#17573026, country#17573027, supplier_type#17573028, declaring_supplier_id#17573029, canonical_id#17573030, is_primary#17573031, threshold#17573032, create_date#17573033], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SEL...
org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | Unknown | Unknown |
0/1
| ||||
| 35022 | default | count at <unknown>:0 org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | Unknown | Unknown |
0/0
| ||||
| 35020 | default | collect at /opt/app/scheduler/jobs/dedup_job.py:386
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Filter isnotnull(canonical_id#17573030)
+- InMemoryTableScan [brand_context_id#17573022, supplier_id#17573023, canonical_id#17573030, is_primary#17573031, entity_name#17573024, city#17573025, state#17573026, country#17573027, threshold#17573032, create_date#17573033], [isnotnull(canonical_id#17573030)]
+- InMemoryRelation [brand_context_id#17573022, supplier_id#17573023, entity_name#17573024, city#17573025, state#17573026, country#17573027, supplier_type#17573028, declaring_supplier_id#17573029, canonical_id#17573030, is_primary#17573031, threshold#17573032, create_date#17573033], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
...
org.apache.spark.sql.Dataset.collectToPython(Dataset.scala:4145) jdk.internal.reflect.GeneratedMethodAccessor272.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | Unknown | Unknown |
0/21
| ||||
| 35019 | default | count at <unknown>:0
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Project [brand_context_id#17573022, supplier_id#17573023, entity_name#17573024, city#17573025, state#17573026, country#17573027, supplier_type#17573028, declaring_supplier_id#17573029, threshold#17573032, create_date#17573033]
+- Filter isnull(canonical_id#17573030)
+- InMemoryTableScan [brand_context_id#17573022, canonical_id#17573030, city#17573025, country#17573027, create_date#17573033, declaring_supplier_id#17573029, entity_name#17573024, state#17573026, supplier_id#17573023, supplier_type#17573028, threshold#17573032], [isnull(canonical_id#17573030)]
+- InMemoryRelation [brand_context_id#17573022, supplier_id#17573023, entity_name#17573024, city#17573025, state#17573026, country#17573027, supplier_type#17573028, declaring_supplier_id#17573029, canonical_id#17573030, is_primary#17573031, threshold#17573032, create_date#17573033], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SEL...
org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | Unknown | Unknown |
0/1
| ||||
| 35018 | default | count at <unknown>:0 org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | Unknown | Unknown |
0/0
| ||||
| 35016 | default | count at <unknown>:0
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Project [brand_context_id#17573022, supplier_id#17573023, entity_name#17573024, city#17573025, state#17573026, country#17573027, supplier_type#17573028, declaring_supplier_id#17573029, threshold#17573032, create_date#17573033]
+- Filter isnull(canonical_id#17573030)
+- InMemoryTableScan [brand_context_id#17573022, canonical_id#17573030, city#17573025, country#17573027, create_date#17573033, declaring_supplier_id#17573029, entity_name#17573024, state#17573026, supplier_id#17573023, supplier_type#17573028, threshold#17573032], [isnull(canonical_id#17573030)]
+- InMemoryRelation [brand_context_id#17573022, supplier_id#17573023, entity_name#17573024, city#17573025, state#17573026, country#17573027, supplier_type#17573028, declaring_supplier_id#17573029, canonical_id#17573030, is_primary#17573031, threshold#17573032, create_date#17573033], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SEL...
org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | Unknown | Unknown |
0/1
| ||||
| 35015 | default | count at <unknown>:0 org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | Unknown | Unknown |
0/0
| ||||
| 35013 | default | count at <unknown>:0
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Filter isnotnull(canonical_id#17573030)
+- InMemoryTableScan [brand_context_id#17573022, supplier_id#17573023, canonical_id#17573030, is_primary#17573031, entity_name#17573024, city#17573025, state#17573026, country#17573027, threshold#17573032, create_date#17573033], [isnotnull(canonical_id#17573030)]
+- InMemoryRelation [brand_context_id#17573022, supplier_id#17573023, entity_name#17573024, city#17573025, state#17573026, country#17573027, supplier_type#17573028, declaring_supplier_id#17573029, canonical_id#17573030, is_primary#17573031, threshold#17573032, create_date#17573033], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
...
org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | Unknown | Unknown |
0/21
| ||||
| 35012 | default | count at <unknown>:0
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Project [brand_context_id#17573022, supplier_id#17573023, entity_name#17573024, city#17573025, state#17573026, country#17573027, supplier_type#17573028, declaring_supplier_id#17573029, threshold#17573032, create_date#17573033]
+- Filter isnull(canonical_id#17573030)
+- InMemoryTableScan [brand_context_id#17573022, canonical_id#17573030, city#17573025, country#17573027, create_date#17573033, declaring_supplier_id#17573029, entity_name#17573024, state#17573026, supplier_id#17573023, supplier_type#17573028, threshold#17573032], [isnull(canonical_id#17573030)]
+- InMemoryRelation [brand_context_id#17573022, supplier_id#17573023, entity_name#17573024, city#17573025, state#17573026, country#17573027, supplier_type#17573028, declaring_supplier_id#17573029, canonical_id#17573030, is_primary#17573031, threshold#17573032, create_date#17573033], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SEL...
org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | Unknown | Unknown |
0/1
| ||||
| 35011 | default | count at <unknown>:0 org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | Unknown | Unknown |
0/0
| ||||
| 35009 | default | count at <unknown>:0
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Project [brand_context_id#17573022, supplier_id#17573023, entity_name#17573024, city#17573025, state#17573026, country#17573027, supplier_type#17573028, declaring_supplier_id#17573029, threshold#17573032, create_date#17573033]
+- Filter isnull(canonical_id#17573030)
+- InMemoryTableScan [brand_context_id#17573022, canonical_id#17573030, city#17573025, country#17573027, create_date#17573033, declaring_supplier_id#17573029, entity_name#17573024, state#17573026, supplier_id#17573023, supplier_type#17573028, threshold#17573032], [isnull(canonical_id#17573030)]
+- InMemoryRelation [brand_context_id#17573022, supplier_id#17573023, entity_name#17573024, city#17573025, state#17573026, country#17573027, supplier_type#17573028, declaring_supplier_id#17573029, canonical_id#17573030, is_primary#17573031, threshold#17573032, create_date#17573033], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SEL...
org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | Unknown | Unknown |
0/1
| ||||
| 35008 | default | count at <unknown>:0 org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | Unknown | Unknown |
0/0
| ||||
| 35006 | default | count at <unknown>:0 org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | Unknown | Unknown |
0/0
| ||||
| 35005 | default | count at <unknown>:0
RDD: AdaptiveSparkPlan isFinalPlan=false
+- Project [brand_context_id#17573022, supplier_id#17573023, entity_name#17573024, city#17573025, state#17573026, country#17573027, supplier_type#17573028, declaring_supplier_id#17573029, threshold#17573032, create_date#17573033]
+- Filter isnull(canonical_id#17573030)
+- InMemoryTableScan [brand_context_id#17573022, canonical_id#17573030, city#17573025, country#17573027, create_date#17573033, declaring_supplier_id#17573029, entity_name#17573024, state#17573026, supplier_id#17573023, supplier_type#17573028, threshold#17573032], [isnull(canonical_id#17573030)]
+- InMemoryRelation [brand_context_id#17573022, supplier_id#17573023, entity_name#17573024, city#17573025, state#17573026, country#17573027, supplier_type#17573028, declaring_supplier_id#17573029, canonical_id#17573030, is_primary#17573031, threshold#17573032, create_date#17573033], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SEL...
org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor271.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source) | Unknown | Unknown |
0/1
| ||||
| 35003 | default | $anonfun$withThreadLocalCaptured$1 at <unknown>:0 org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source) | Unknown | Unknown |
0/0
|