DataHubScheduler - Details for Stage 39732 (Attempt 0)

Resource Profile Id: 0
Total Time Across All Tasks: 0.9 s
Locality Level Summary: Node local: 1
Input Size / Records: 17.2 KiB / 13
Associated Job Ids: 25151

digraph G { subgraph clusterstage_39732 { label="Stage 39732"; 169587 [labelType="html" label="PythonRDD [169587] RDD at PythonRDD.scala:53"]; 170205 [labelType="html" label="SQLExecutionRDD [170205] [Unordered] javaToPython at <unknown>:0"]; 170208 [labelType="html" label="PythonRDD [170208] [Unordered] foreachPartition at /opt/app/scheduler/jobs/memgraph_loader_job.py:2107"]; subgraph cluster403957 { label="parallelize"; 169586 [labelType="html" label="ParallelCollectionRDD [169586] readRDDFromFile at PythonRDD.scala:289"]; } subgraph cluster403958 { label="mapPartitions"; 169588 [labelType="html" label="MapPartitionsRDD [169588] mapPartitions at SerDeUtil.scala:117"]; } subgraph cluster403959 { label="map"; 169589 [labelType="html" label="MapPartitionsRDD [169589] map at SerDeUtil.scala:69"]; } subgraph cluster403960 { label="mapPartitions"; 169590 [labelType="html" label="MapPartitionsRDD [169590] applySchemaToPythonRDD at <unknown>:0"]; } subgraph cluster405020 { label="Exchange"; 170042 [labelType="html" label="ShuffledRowRDD [170042] [Unordered] foreachPartition at /opt/app/scheduler/jobs/memgraph_loader_job.py:2107"]; } subgraph cluster405014 { label="WholeStageCodegen (10)"; 170043 [labelType="html" label="MapPartitionsRDD [170043] [Unordered] foreachPartition at /opt/app/scheduler/jobs/memgraph_loader_job.py:2107"]; } subgraph cluster405029 { label="Exchange"; 170044 [labelType="html" label="ShuffledRowRDD [170044] [Unordered] foreachPartition at /opt/app/scheduler/jobs/memgraph_loader_job.py:2107"]; } subgraph cluster405021 { label="WholeStageCodegen (11)"; 170045 [labelType="html" label="MapPartitionsRDD [170045] [Unordered] foreachPartition at /opt/app/scheduler/jobs/memgraph_loader_job.py:2107"]; } subgraph cluster405013 { label="Union"; 170046 [labelType="html" label="UnionRDD [170046] [Unordered] foreachPartition at /opt/app/scheduler/jobs/memgraph_loader_job.py:2107"]; } subgraph cluster404800 { label="AdaptiveSparkPlan"; 170047 [labelType="html" label="MapPartitionsRDD [170047] [Unordered] foreachPartition at /opt/app/scheduler/jobs/memgraph_loader_job.py:2107"]; 170048 [labelType="html" label="AdaptiveSparkPlan isFinalPlan=false\n+- Union\n :- Project [sid#20346246 AS entity_id#20350117, 6a8bf0ac7cc171297135c303 AS brand_context_id#20350118, dim_id#20344359, cast(lvl#20349326 as string) AS level#20350119, CASE WHEN (lvl#20349326 >= 4.2) THEN Extreme WHEN (lvl#20349326 >= 3.4) THEN High WHEN (lvl#20349326 >= 2.6) THEN Moderate WHEN (lvl#20349326 >= 1.8) THEN Medium WHEN (lvl#20349326 >= 1.0) THEN Low ELSE Unknown END AS level_band#20350120]\n : +- Filter (isnotnull(lvl#20349326) AND (lvl#20349326 > 0.0))\n : +- HashAggregate(keys=[sid#20346246, dim_id#20344359], functions=[max(lvl#20346241)], output=[sid#20346246, dim_id#20344359, lvl#20349326])\n : +- Exchange hashpartitioning(sid#20346246, dim_id#20344359, 10), ENSURE_REQUIREMENTS, [plan_id=3320001]\n : +- HashAggregate(keys=[sid#20346246, dim_id#20344359], functions=[partial_max(lvl#20346241)], output=[sid#20346246, dim_id#20344359, max#20350555])\n : +- Union\n : :- InMemoryTableScan [s... [170048] [Cached] [Unordered] foreachPartition at /opt/app/scheduler/jobs/memgraph_loader_job.py:2107"]; } subgraph cluster405507 { label="InMemoryTableScan"; 170197 [labelType="html" label="MapPartitionsRDD [170197] [Unordered] javaToPython at <unknown>:0"]; 170198 [labelType="html" label="MapPartitionsRDD [170198] [Unordered] javaToPython at <unknown>:0"]; 170199 [labelType="html" label="MapPartitionsRDD [170199] [Unordered] javaToPython at <unknown>:0"]; } subgraph cluster405502 { label="WholeStageCodegen (15)"; 170200 [labelType="html" label="MapPartitionsRDD [170200] [Unordered] javaToPython at <unknown>:0"]; } subgraph cluster405508 { label="WholeStageCodegen (16)"; 170201 [labelType="html" label="MapPartitionsRDD [170201] javaToPython at <unknown>:0"]; } subgraph cluster405501 { label="Union"; 170202 [labelType="html" label="UnionRDD [170202] [Unordered] javaToPython at <unknown>:0"]; } subgraph cluster405498 { label="WholeStageCodegen (17)"; 170203 [labelType="html" label="MapPartitionsRDD [170203] [Unordered] javaToPython at <unknown>:0"]; } subgraph cluster405497 { label="Coalesce"; 170204 [labelType="html" label="CoalescedRDD [170204] [Unordered] javaToPython at <unknown>:0"]; } subgraph cluster405513 { label="map"; 170206 [labelType="html" label="MapPartitionsRDD [170206] [Unordered] javaToPython at <unknown>:0"]; } subgraph cluster405514 { label="mapPartitions"; 170207 [labelType="html" label="MapPartitionsRDD [170207] [Unordered] javaToPython at <unknown>:0"]; } } 169586->169587; 169587->169588; 169588->169589; 169589->169590; 170042->170043; 170044->170045; 170043->170046; 170045->170046; 170046->170047; 170047->170048; 170048->170197; 170197->170198; 170198->170199; 170199->170200; 169590->170201; 170200->170202; 170201->170202; 170202->170203; 170203->170204; 170204->170205; 170205->170206; 170206->170207; 170207->170208; }

170007,170042

170041,170044

170048

Event Timeline

Enable zooming