DataHubScheduler - Details for Stage 41738 (Attempt 0)

Resource Profile Id: 0
Total Time Across All Tasks: 1.0 s
Locality Level Summary: Node local: 1
Input Size / Records: 14.1 KiB / 11
Shuffle Read Size / Records: 4.7 KiB / 37
Associated Job Ids: 30867

digraph G { subgraph clusterstage_41738 { label="Stage 41738"; 182820 [labelType="html" label="PythonRDD [182820] RDD at PythonRDD.scala:53"]; 183191 [labelType="html" label="SQLExecutionRDD [183191] [Unordered] javaToPython at <unknown>:0"]; 183194 [labelType="html" label="PythonRDD [183194] [Unordered] foreachPartition at /opt/app/scheduler/jobs/memgraph_loader_job.py:2107"]; subgraph cluster431235 { label="parallelize"; 182819 [labelType="html" label="ParallelCollectionRDD [182819] readRDDFromFile at PythonRDD.scala:289"]; } subgraph cluster431236 { label="mapPartitions"; 182821 [labelType="html" label="MapPartitionsRDD [182821] mapPartitions at SerDeUtil.scala:117"]; } subgraph cluster431237 { label="map"; 182822 [labelType="html" label="MapPartitionsRDD [182822] map at SerDeUtil.scala:69"]; } subgraph cluster431238 { label="mapPartitions"; 182823 [labelType="html" label="MapPartitionsRDD [182823] applySchemaToPythonRDD at <unknown>:0"]; } subgraph cluster431870 { label="Exchange"; 183098 [labelType="html" label="ShuffledRowRDD [183098] [Unordered] javaToPython at <unknown>:0"]; } subgraph cluster431864 { label="WholeStageCodegen (10)"; 183099 [labelType="html" label="MapPartitionsRDD [183099] [Unordered] javaToPython at <unknown>:0"]; } subgraph cluster431879 { label="Exchange"; 183100 [labelType="html" label="ShuffledRowRDD [183100] [Unordered] javaToPython at <unknown>:0"]; } subgraph cluster431871 { label="WholeStageCodegen (11)"; 183101 [labelType="html" label="MapPartitionsRDD [183101] [Unordered] javaToPython at <unknown>:0"]; } subgraph cluster431863 { label="Union"; 183102 [labelType="html" label="UnionRDD [183102] [Unordered] javaToPython at <unknown>:0"]; } subgraph cluster431735 { label="AdaptiveSparkPlan"; 183103 [labelType="html" label="MapPartitionsRDD [183103] [Unordered] javaToPython at <unknown>:0"]; 183104 [labelType="html" label="AdaptiveSparkPlan isFinalPlan=false\n+- Union\n :- Project [sid#16674975 AS entity_id#16676746, 6a72fd3a7e0eaa616bfdc3fe AS brand_context_id#16676747, dim_id#16673172, cast(lvl#16675955 as string) AS level#16676748, CASE WHEN (lvl#16675955 >= 4.2) THEN Extreme WHEN (lvl#16675955 >= 3.4) THEN High WHEN (lvl#16675955 >= 2.6) THEN Moderate WHEN (lvl#16675955 >= 1.8) THEN Medium WHEN (lvl#16675955 >= 1.0) THEN Low ELSE Unknown END AS level_band#16676749]\n : +- Filter (isnotnull(lvl#16675955) AND (lvl#16675955 > 0.0))\n : +- HashAggregate(keys=[sid#16674975, dim_id#16673172], functions=[max(lvl#16674971)], output=[sid#16674975, dim_id#16673172, lvl#16675955])\n : +- Exchange hashpartitioning(sid#16674975, dim_id#16673172, 10), ENSURE_REQUIREMENTS, [plan_id=4524618]\n : +- HashAggregate(keys=[sid#16674975, dim_id#16673172], functions=[partial_max(lvl#16674971)], output=[sid#16674975, dim_id#16673172, max#16677184])\n : +- Union\n : :- InMemoryTableScan [s... [183104] [Cached] [Unordered] javaToPython at <unknown>:0"]; } subgraph cluster432145 { label="AQEShuffleRead"; 183178 [labelType="html" label="ShuffledRowRDD [183178] [Unordered] javaToPython at <unknown>:0"]; } subgraph cluster432141 { label="WholeStageCodegen (18)"; 183179 [labelType="html" label="MapPartitionsRDD [183179] [Unordered] javaToPython at <unknown>:0"]; } subgraph cluster432151 { label="AQEShuffleRead"; 183180 [labelType="html" label="ShuffledRowRDD [183180] [Unordered] javaToPython at <unknown>:0"]; } subgraph cluster432146 { label="WholeStageCodegen (19)"; 183181 [labelType="html" label="MapPartitionsRDD [183181] [Unordered] javaToPython at <unknown>:0"]; } subgraph cluster432140 { label="Union"; 183182 [labelType="html" label="UnionRDD [183182] [Unordered] javaToPython at <unknown>:0"]; } subgraph cluster432132 { label="WholeStageCodegen (20)"; 183183 [labelType="html" label="MapPartitionsRDD [183183] [Unordered] javaToPython at <unknown>:0"]; } subgraph cluster432160 { label="InMemoryTableScan"; 183184 [labelType="html" label="MapPartitionsRDD [183184] [Unordered] javaToPython at <unknown>:0"]; 183185 [labelType="html" label="MapPartitionsRDD [183185] [Unordered] javaToPython at <unknown>:0"]; 183186 [labelType="html" label="MapPartitionsRDD [183186] [Unordered] javaToPython at <unknown>:0"]; } subgraph cluster432152 { label="WholeStageCodegen (21)"; 183187 [labelType="html" label="MapPartitionsRDD [183187] [Unordered] javaToPython at <unknown>:0"]; } subgraph cluster432161 { label="WholeStageCodegen (22)"; 183188 [labelType="html" label="MapPartitionsRDD [183188] javaToPython at <unknown>:0"]; } subgraph cluster432131 { label="Union"; 183189 [labelType="html" label="UnionRDD [183189] [Unordered] javaToPython at <unknown>:0"]; } subgraph cluster432130 { label="Coalesce"; 183190 [labelType="html" label="CoalescedRDD [183190] [Unordered] javaToPython at <unknown>:0"]; } subgraph cluster432169 { label="map"; 183192 [labelType="html" label="MapPartitionsRDD [183192] [Unordered] javaToPython at <unknown>:0"]; } subgraph cluster432170 { label="mapPartitions"; 183193 [labelType="html" label="MapPartitionsRDD [183193] [Unordered] javaToPython at <unknown>:0"]; } } 182819->182820; 182820->182821; 182821->182822; 182822->182823; 183098->183099; 183100->183101; 183099->183102; 183101->183102; 183102->183103; 183103->183104; 183178->183179; 183180->183181; 183179->183182; 183181->183182; 183182->183183; 183104->183184; 183184->183185; 183185->183186; 183186->183187; 182823->183188; 183183->183189; 183187->183189; 183188->183189; 183189->183190; 183190->183191; 183191->183192; 183192->183193; 183193->183194; }

183078,183098

183097,183100

183169,183178

183177,183180

183104

Event Timeline

Enable zooming