DataHubScheduler - Details for Query 5767

Submitted Time: 2026/10/16 08:19:53
Duration: 6 s
Succeeded Jobs: 8811

Show the Stage ID and Task ID that corresponds to the max metric

digraph G { 0 [labelType="html" label=" AdaptiveSparkPlan "]; 1 [labelType="html" label=" DeserializeToObject "]; 2 [labelType="html" label="InMemoryTableScan number of output rows: 10,000"]; subgraph cluster3 { isCluster="true"; label="WholeStageCodegen (1)\n \nduration: 0 ms"; 4 [labelType="html" label="Scan JDBCRelation((SELECT * FROM dbo.dimcity WHERE bigid BETWEEN 40001 AND 50000) AS page) [numPartitions=1] number of output rows: 0"]; } 1->0; 2->1; 4->2; }

AdaptiveSparkPlan isFinalPlan=true

DeserializeToObject createexternalrow(staticinvoke(class java.lang.Integer, ObjectType(class java.lang.Integer), valueOf, _id#10402760, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Timestamp), toJavaTimestamp, create_ts#10402761, true, false, true), staticinvoke(class java.lang.Double, ObjectType(class java.lang.Double), valueOf, entity_version#10402762, true, false, true), staticinvoke(class java.lang.Double, ObjectType(class java.lang.Double), valueOf, lat#10402763, true, false, true), staticinvoke(class java.lang.Double, ObjectType(class java.lang.Double), valueOf, lon#10402764, true, false, true), name#10402765.toString, staticinvoke(class java.lang.Integer, ObjectType(class java.lang.Integer), valueOf, state_id#10402766, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Timestamp), toJavaTimestamp, update_ts#10402767, true, false, true), file_name#10402768.toString, staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Date), toJavaDate, create_date#10402769, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Date), toJavaDate, update_date#10402770, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Timestamp), toJavaTimestamp, ETL_runtime#10402771, true, false, true), StructField(_id,IntegerType,true), StructField(create_ts,TimestampType,true), StructField(entity_version,DoubleType,true), StructField(lat,DoubleType,true), StructField(lon,DoubleType,true), StructField(name,StringType,true), StructField(state_id,IntegerType,true), StructField(update_ts,TimestampType,true), StructField(file_name,StringType,true), StructField(create_date,DateType,true), StructField(update_date,DateType,true), StructField(ETL_runtime,TimestampType,true)), obj#10410689: org.apache.spark.sql.Row

InMemoryTableScan [_id#10402760, create_ts#10402761, entity_version#10402762, lat#10402763, lon#10402764, name#10402765, state_id#10402766, update_ts#10402767, file_name#10402768, create_date#10402769, update_date#10402770, ETL_runtime#10402771]

Scan JDBCRelation((SELECT * FROM dbo.dimcity WHERE bigid BETWEEN 40001 AND 50000) AS page) [numPartitions=1] [_id#10402760,create_ts#10402761,entity_version#10402762,lat#10402763,lon#10402764,name#10402765,state_id#10402766,update_ts#10402767,file_name#10402768,create_date#10402769,update_date#10402770,ETL_runtime#10402771,bigid#10402772L] PushedFilters: [], ReadSchema: struct<_id:int,create_ts:timestamp,entity_version:double,lat:double,lon:double,name:string,state_...

WholeStageCodegen (1)

Details

== Physical Plan ==
AdaptiveSparkPlan (7)
+- == Final Plan ==
   DeserializeToObject (5)
   +- TableCacheQueryStage (4), Statistics(sizeInBytes=865.8 KiB, rowCount=1.00E+4)
      +- InMemoryTableScan (1)
            +- InMemoryRelation (2)
                  +- * Scan JDBCRelation((SELECT * FROM dbo.dimcity WHERE bigid BETWEEN 40001 AND 50000) AS page) [numPartitions=1]  (3)
+- == Initial Plan ==
   DeserializeToObject (6)
   +- InMemoryTableScan (1)
         +- InMemoryRelation (2)
               +- * Scan JDBCRelation((SELECT * FROM dbo.dimcity WHERE bigid BETWEEN 40001 AND 50000) AS page) [numPartitions=1]  (3)


(1) InMemoryTableScan
Output [12]: [_id#10402760, create_ts#10402761, entity_version#10402762, lat#10402763, lon#10402764, name#10402765, state_id#10402766, update_ts#10402767, file_name#10402768, create_date#10402769, update_date#10402770, ETL_runtime#10402771]
Arguments: [_id#10402760, create_ts#10402761, entity_version#10402762, lat#10402763, lon#10402764, name#10402765, state_id#10402766, update_ts#10402767, file_name#10402768, create_date#10402769, update_date#10402770, ETL_runtime#10402771]

(2) InMemoryRelation
Arguments: [_id#10402760, create_ts#10402761, entity_version#10402762, lat#10402763, lon#10402764, name#10402765, state_id#10402766, update_ts#10402767, file_name#10402768, create_date#10402769, update_date#10402770, ETL_runtime#10402771, bigid#10402772L], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@7795a5ad,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Scan JDBCRelation((SELECT * FROM dbo.dimcity WHERE bigid BETWEEN 40001 AND 50000) AS page) [numPartitions=1] [_id#10402760,create_ts#10402761,entity_version#10402762,lat#10402763,lon#10402764,name#10402765,state_id#10402766,update_ts#10402767,file_name#10402768,create_date#10402769,update_date#10402770,ETL_runtime#10402771,bigid#10402772L] PushedFilters: [], ReadSchema: struct<_id:int,create_ts:timestamp,entity_version:double,lat:double,lon:double,name:string,state_...
,None)

(3) Scan JDBCRelation((SELECT * FROM dbo.dimcity WHERE bigid BETWEEN 40001 AND 50000) AS page) [numPartitions=1]  [codegen id : 1]
Output [13]: [_id#10402760, create_ts#10402761, entity_version#10402762, lat#10402763, lon#10402764, name#10402765, state_id#10402766, update_ts#10402767, file_name#10402768, create_date#10402769, update_date#10402770, ETL_runtime#10402771, bigid#10402772L]
ReadSchema: struct<_id:int,create_ts:timestamp,entity_version:double,lat:double,lon:double,name:string,state_id:int,update_ts:timestamp,file_name:string,create_date:date,update_date:date,ETL_runtime:timestamp,bigid:bigint>

(4) TableCacheQueryStage
Output [12]: [_id#10402760, create_ts#10402761, entity_version#10402762, lat#10402763, lon#10402764, name#10402765, state_id#10402766, update_ts#10402767, file_name#10402768, create_date#10402769, update_date#10402770, ETL_runtime#10402771]
Arguments: 0

(5) DeserializeToObject
Input [12]: [_id#10402760, create_ts#10402761, entity_version#10402762, lat#10402763, lon#10402764, name#10402765, state_id#10402766, update_ts#10402767, file_name#10402768, create_date#10402769, update_date#10402770, ETL_runtime#10402771]
Arguments: createexternalrow(staticinvoke(class java.lang.Integer, ObjectType(class java.lang.Integer), valueOf, _id#10402760, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Timestamp), toJavaTimestamp, create_ts#10402761, true, false, true), staticinvoke(class java.lang.Double, ObjectType(class java.lang.Double), valueOf, entity_version#10402762, true, false, true), staticinvoke(class java.lang.Double, ObjectType(class java.lang.Double), valueOf, lat#10402763, true, false, true), staticinvoke(class java.lang.Double, ObjectType(class java.lang.Double), valueOf, lon#10402764, true, false, true), name#10402765.toString, staticinvoke(class java.lang.Integer, ObjectType(class java.lang.Integer), valueOf, state_id#10402766, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Timestamp), toJavaTimestamp, update_ts#10402767, true, false, true), file_name#10402768.toString, staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Date), toJavaDate, create_date#10402769, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Date), toJavaDate, update_date#10402770, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Timestamp), toJavaTimestamp, ETL_runtime#10402771, true, false, true), StructField(_id,IntegerType,true), StructField(create_ts,TimestampType,true), StructField(entity_version,DoubleType,true), StructField(lat,DoubleType,true), StructField(lon,DoubleType,true), StructField(name,StringType,true), StructField(state_id,IntegerType,true), StructField(update_ts,TimestampType,true), StructField(file_name,StringType,true), StructField(create_date,DateType,true), StructField(update_date,DateType,true), StructField(ETL_runtime,TimestampType,true)), obj#10410689: org.apache.spark.sql.Row

(6) DeserializeToObject
Input [12]: [_id#10402760, create_ts#10402761, entity_version#10402762, lat#10402763, lon#10402764, name#10402765, state_id#10402766, update_ts#10402767, file_name#10402768, create_date#10402769, update_date#10402770, ETL_runtime#10402771]
Arguments: createexternalrow(staticinvoke(class java.lang.Integer, ObjectType(class java.lang.Integer), valueOf, _id#10402760, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Timestamp), toJavaTimestamp, create_ts#10402761, true, false, true), staticinvoke(class java.lang.Double, ObjectType(class java.lang.Double), valueOf, entity_version#10402762, true, false, true), staticinvoke(class java.lang.Double, ObjectType(class java.lang.Double), valueOf, lat#10402763, true, false, true), staticinvoke(class java.lang.Double, ObjectType(class java.lang.Double), valueOf, lon#10402764, true, false, true), name#10402765.toString, staticinvoke(class java.lang.Integer, ObjectType(class java.lang.Integer), valueOf, state_id#10402766, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Timestamp), toJavaTimestamp, update_ts#10402767, true, false, true), file_name#10402768.toString, staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Date), toJavaDate, create_date#10402769, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Date), toJavaDate, update_date#10402770, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Timestamp), toJavaTimestamp, ETL_runtime#10402771, true, false, true), StructField(_id,IntegerType,true), StructField(create_ts,TimestampType,true), StructField(entity_version,DoubleType,true), StructField(lat,DoubleType,true), StructField(lon,DoubleType,true), StructField(name,StringType,true), StructField(state_id,IntegerType,true), StructField(update_ts,TimestampType,true), StructField(file_name,StringType,true), StructField(create_date,DateType,true), StructField(update_date,DateType,true), StructField(ETL_runtime,TimestampType,true)), obj#10410689: org.apache.spark.sql.Row

(7) AdaptiveSparkPlan
Output [1]: [obj#10410689]
Arguments: isFinalPlan=true

SQL / DataFrame Properties

Name	Value
spark.sql.shuffle.partitions	10