DataHubScheduler - Details for Job 24375

Status: SUCCEEDED
Submitted: 2026/10/18 12:02:56
Duration: 54 ms
Associated SQL Query: 5528
Completed Stages: 1
Skipped Stages: 1

Enable zooming

digraph G { subgraph clusterstage_34170 { label="Stage 34170 (skipped)"; 150226 [labelType="html" label="JDBCRDD [150226] cache at <unknown>:0"]; subgraph cluster362988 { label="WholeStageCodegen (1)"; 150227 [labelType="html" label="MapPartitionsRDD [150227] count at <unknown>:0"]; } subgraph cluster362990 { label="mapPartitionsInternal"; 150228 [labelType="html" label="MapPartitionsRDD [150228] count at <unknown>:0"]; } subgraph cluster362991 { label="mapPartitionsInternal"; 150229 [labelType="html" label="*(1) Scan JDBCRelation((\n SELECT DISTINCT\n v.brand_context_id,\n v.supplier_id,\n v.entity_name,\n v.city,\n v.state,\n v.country,\n v.supplier_type,\n v.declaring_supplier_id,\n v.canonical_id,\n v.is_primary,\n COALESCE(v.threshold, 0.9) AS threshold,\n v.create_date\n FROM vw_all_suppliers_for_dedup v\n ) AS all_suppliers) [numPartitions=1] [brand_context_id#14337219,supplier_id#14337220,entity_name#14337221,city#14337222,state#14337223,country#14337224,supplier_type#14337225,declaring_supplier_id#14337226,canonical_id#14337227,is_primary#14337228,threshold#14337229,create_date#14337230] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...\n [150229] [Cached] count at <unknown>:0"]; } subgraph cluster363048 { label="InMemoryTableScan"; 150250 [labelType="html" label="MapPartitionsRDD [150250] count at <unknown>:0"]; 150251 [labelType="html" label="MapPartitionsRDD [150251] count at <unknown>:0"]; 150252 [labelType="html" label="MapPartitionsRDD [150252] count at <unknown>:0"]; } subgraph cluster363044 { label="WholeStageCodegen (1)"; 150253 [labelType="html" label="MapPartitionsRDD [150253] count at <unknown>:0"]; } subgraph cluster363049 { label="mapPartitionsInternal"; 150254 [labelType="html" label="MapPartitionsRDD [150254] count at <unknown>:0"]; } subgraph cluster363050 { label="mapPartitionsInternal"; 150255 [labelType="html" label="AdaptiveSparkPlan isFinalPlan=false\n+- Filter isnotnull(canonical_id#14337227)\n +- InMemoryTableScan [brand_context_id#14337219, supplier_id#14337220, canonical_id#14337227, is_primary#14337228, entity_name#14337221, city#14337222, state#14337223, country#14337224, threshold#14337229, create_date#14337230], [isnotnull(canonical_id#14337227)]\n +- InMemoryRelation [brand_context_id#14337219, supplier_id#14337220, entity_name#14337221, city#14337222, state#14337223, country#14337224, supplier_type#14337225, declaring_supplier_id#14337226, canonical_id#14337227, is_primary#14337228, threshold#14337229, create_date#14337230], StorageLevel(disk, memory, deserialized, 1 replicas)\n +- *(1) Scan JDBCRelation((\n SELECT DISTINCT\n v.brand_context_id,\n v.supplier_id,\n v.entity_name,\n v.city,\n v.state,\n v.country,\n v.supplier_type,\n v.declaring_supplier_id,\n ... [150255] [Cached] count at <unknown>:0"]; } subgraph cluster363056 { label="InMemoryTableScan"; 150256 [labelType="html" label="MapPartitionsRDD [150256] count at <unknown>:0"]; 150257 [labelType="html" label="MapPartitionsRDD [150257] count at <unknown>:0"]; 150258 [labelType="html" label="MapPartitionsRDD [150258] count at <unknown>:0"]; } subgraph cluster363052 { label="WholeStageCodegen (1)"; 150259 [labelType="html" label="MapPartitionsRDD [150259] count at <unknown>:0"]; } subgraph cluster363051 { label="Exchange"; 150260 [labelType="html" label="MapPartitionsRDD [150260] count at <unknown>:0"]; } } 150226->150227; 150227->150228; 150228->150229; 150229->150250; 150250->150251; 150251->150252; 150252->150253; 150253->150254; 150254->150255; 150255->150256; 150256->150257; 150257->150258; 150258->150259; 150259->150260; }

150229

150255

digraph G { subgraph clusterstage_34171 { label="Stage 34171"; subgraph cluster363061 { label="Exchange"; 150261 [labelType="html" label="ShuffledRowRDD [150261] [Unordered] count at <unknown>:0"]; } subgraph cluster363057 { label="WholeStageCodegen (2)"; 150262 [labelType="html" label="MapPartitionsRDD [150262] [Unordered] count at <unknown>:0"]; } subgraph cluster363062 { label="mapPartitionsInternal"; 150263 [labelType="html" label="MapPartitionsRDD [150263] [Unordered] count at <unknown>:0"]; } } 150261->150262; 150262->150263; }

150260,150261

Completed Stages (1)

Page:

Stage Id ▾	Pool Name	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
34171	default	count at <unknown>:0 +details org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor202.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source)	2026/10/18 12:02:56	52 ms	1/1			59.0 B

Stage Id ▾

Pool Name

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

34171

default

count at <unknown>:0 +details

org.apache.spark.sql.Dataset.count(Dataset.scala:3613)
jdk.internal.reflect.GeneratedMethodAccessor202.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
py4j.ClientServerConnection.run(ClientServerConnection.java:106)
java.base/java.lang.Thread.run(Unknown Source)

2026/10/18 12:02:56

52 ms

1/1

59.0 B

Page:

Skipped Stages (1)

Page:

Stage Id ▾	Pool Name	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
34170	default	count at <unknown>:0 +details org.apache.spark.sql.Dataset.count(Dataset.scala:3613) jdk.internal.reflect.GeneratedMethodAccessor202.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source)	Unknown	Unknown	0/1

Stage Id ▾

Pool Name

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

34170

default

count at <unknown>:0 +details

org.apache.spark.sql.Dataset.count(Dataset.scala:3613)
jdk.internal.reflect.GeneratedMethodAccessor202.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
py4j.ClientServerConnection.run(ClientServerConnection.java:106)
java.base/java.lang.Thread.run(Unknown Source)

Unknown

0/1

Page: