== Physical Plan ==
AdaptiveSparkPlan (155)
+- == Final Plan ==
* HashAggregate (151)
+- ShuffleQueryStage (150), Statistics(sizeInBytes=336.0 B, rowCount=21)
+- Exchange (149)
+- * HashAggregate (148)
+- TableCacheQueryStage (147), Statistics(sizeInBytes=178.0 B, rowCount=1)
+- InMemoryTableScan (1)
+- InMemoryRelation (2)
+- AdaptiveSparkPlan (146)
+- Union (145)
:- Project (23)
: +- BroadcastHashJoin Inner BuildLeft (22)
: :- BroadcastExchange (12)
: : +- Filter (11)
: : +- InMemoryTableScan (3)
: : +- InMemoryRelation (4)
: : +- AdaptiveSparkPlan (10)
: : +- Project (9)
: : +- Filter (8)
: : +- InMemoryTableScan (5)
: : +- InMemoryRelation (6)
: : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: +- Project (21)
: +- Filter (20)
: +- InMemoryTableScan (13)
: +- InMemoryRelation (14)
: +- AdaptiveSparkPlan (19)
: +- Filter (18)
: +- InMemoryTableScan (15)
: +- InMemoryRelation (16)
: +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (17)
:- Project (72)
: +- Filter (71)
: +- Window (70)
: +- WindowGroupLimit (69)
: +- Sort (68)
: +- Exchange (67)
: +- WindowGroupLimit (66)
: +- Sort (65)
: +- Project (64)
: +- BatchEvalPython (63)
: +- Project (62)
: +- Filter (61)
: +- BatchEvalPython (60)
: +- Project (59)
: +- BroadcastHashJoin Inner BuildLeft (58)
: :- BroadcastExchange (51)
: : +- Project (50)
: : +- SortMergeJoin LeftAnti (49)
: : :- Sort (34)
: : : +- Exchange (33)
: : : +- Project (32)
: : : +- BatchEvalPython (31)
: : : +- Project (30)
: : : +- Filter (29)
: : : +- BatchEvalPython (28)
: : : +- Filter (27)
: : : +- InMemoryTableScan (24)
: : : +- InMemoryRelation (25)
: : : +- AdaptiveSparkPlan (26)
: : : +- Project (9)
: : : +- Filter (8)
: : : +- InMemoryTableScan (5)
: : : +- InMemoryRelation (6)
: : : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: : +- Sort (48)
: : +- Exchange (47)
: : +- Project (46)
: : +- BroadcastHashJoin Inner BuildLeft (45)
: : :- BroadcastExchange (39)
: : : +- Filter (38)
: : : +- InMemoryTableScan (35)
: : : +- InMemoryRelation (36)
: : : +- AdaptiveSparkPlan (37)
: : : +- Project (9)
: : : +- Filter (8)
: : : +- InMemoryTableScan (5)
: : : +- InMemoryRelation (6)
: : : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: : +- Project (44)
: : +- Filter (43)
: : +- InMemoryTableScan (40)
: : +- InMemoryRelation (41)
: : +- AdaptiveSparkPlan (42)
: : +- Filter (18)
: : +- InMemoryTableScan (15)
: : +- InMemoryRelation (16)
: : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (17)
: +- Project (57)
: +- BatchEvalPython (56)
: +- Filter (55)
: +- InMemoryTableScan (52)
: +- InMemoryRelation (53)
: +- AdaptiveSparkPlan (54)
: +- Filter (18)
: +- InMemoryTableScan (15)
: +- InMemoryRelation (16)
: +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (17)
+- Project (144)
+- BroadcastHashJoin LeftAnti BuildRight (143)
:- SortMergeJoin LeftAnti (92)
: :- Sort (77)
: : +- Exchange (76)
: : +- InMemoryTableScan (73)
: : +- InMemoryRelation (74)
: : +- AdaptiveSparkPlan (75)
: : +- Project (9)
: : +- Filter (8)
: : +- InMemoryTableScan (5)
: : +- InMemoryRelation (6)
: : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: +- Sort (91)
: +- Exchange (90)
: +- Project (89)
: +- BroadcastHashJoin Inner BuildLeft (88)
: :- BroadcastExchange (82)
: : +- Filter (81)
: : +- InMemoryTableScan (78)
: : +- InMemoryRelation (79)
: : +- AdaptiveSparkPlan (80)
: : +- Project (9)
: : +- Filter (8)
: : +- InMemoryTableScan (5)
: : +- InMemoryRelation (6)
: : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: +- Project (87)
: +- Filter (86)
: +- InMemoryTableScan (83)
: +- InMemoryRelation (84)
: +- AdaptiveSparkPlan (85)
: +- Filter (18)
: +- InMemoryTableScan (15)
: +- InMemoryRelation (16)
: +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (17)
+- BroadcastExchange (142)
+- Project (141)
+- Filter (140)
+- Window (139)
+- WindowGroupLimit (138)
+- Sort (137)
+- Exchange (136)
+- WindowGroupLimit (135)
+- Sort (134)
+- Project (133)
+- BatchEvalPython (132)
+- Project (131)
+- Filter (130)
+- BatchEvalPython (129)
+- Project (128)
+- BroadcastHashJoin Inner BuildLeft (127)
:- BroadcastExchange (120)
: +- Project (119)
: +- SortMergeJoin LeftAnti (118)
: :- Sort (103)
: : +- Exchange (102)
: : +- Project (101)
: : +- BatchEvalPython (100)
: : +- Project (99)
: : +- Filter (98)
: : +- BatchEvalPython (97)
: : +- Filter (96)
: : +- InMemoryTableScan (93)
: : +- InMemoryRelation (94)
: : +- AdaptiveSparkPlan (95)
: : +- Project (9)
: : +- Filter (8)
: : +- InMemoryTableScan (5)
: : +- InMemoryRelation (6)
: : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: +- Sort (117)
: +- Exchange (116)
: +- Project (115)
: +- BroadcastHashJoin Inner BuildLeft (114)
: :- BroadcastExchange (108)
: : +- Filter (107)
: : +- InMemoryTableScan (104)
: : +- InMemoryRelation (105)
: : +- AdaptiveSparkPlan (106)
: : +- Project (9)
: : +- Filter (8)
: : +- InMemoryTableScan (5)
: : +- InMemoryRelation (6)
: : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: +- Project (113)
: +- Filter (112)
: +- InMemoryTableScan (109)
: +- InMemoryRelation (110)
: +- AdaptiveSparkPlan (111)
: +- Filter (18)
: +- InMemoryTableScan (15)
: +- InMemoryRelation (16)
: +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (17)
+- Project (126)
+- BatchEvalPython (125)
+- Filter (124)
+- InMemoryTableScan (121)
+- InMemoryRelation (122)
+- AdaptiveSparkPlan (123)
+- Filter (18)
+- InMemoryTableScan (15)
+- InMemoryRelation (16)
+- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (17)
+- == Initial Plan ==
HashAggregate (154)
+- Exchange (153)
+- HashAggregate (152)
+- InMemoryTableScan (1)
+- InMemoryRelation (2)
+- AdaptiveSparkPlan (146)
+- Union (145)
:- Project (23)
: +- BroadcastHashJoin Inner BuildLeft (22)
: :- BroadcastExchange (12)
: : +- Filter (11)
: : +- InMemoryTableScan (3)
: : +- InMemoryRelation (4)
: : +- AdaptiveSparkPlan (10)
: : +- Project (9)
: : +- Filter (8)
: : +- InMemoryTableScan (5)
: : +- InMemoryRelation (6)
: : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: +- Project (21)
: +- Filter (20)
: +- InMemoryTableScan (13)
: +- InMemoryRelation (14)
: +- AdaptiveSparkPlan (19)
: +- Filter (18)
: +- InMemoryTableScan (15)
: +- InMemoryRelation (16)
: +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (17)
:- Project (72)
: +- Filter (71)
: +- Window (70)
: +- WindowGroupLimit (69)
: +- Sort (68)
: +- Exchange (67)
: +- WindowGroupLimit (66)
: +- Sort (65)
: +- Project (64)
: +- BatchEvalPython (63)
: +- Project (62)
: +- Filter (61)
: +- BatchEvalPython (60)
: +- Project (59)
: +- BroadcastHashJoin Inner BuildLeft (58)
: :- BroadcastExchange (51)
: : +- Project (50)
: : +- SortMergeJoin LeftAnti (49)
: : :- Sort (34)
: : : +- Exchange (33)
: : : +- Project (32)
: : : +- BatchEvalPython (31)
: : : +- Project (30)
: : : +- Filter (29)
: : : +- BatchEvalPython (28)
: : : +- Filter (27)
: : : +- InMemoryTableScan (24)
: : : +- InMemoryRelation (25)
: : : +- AdaptiveSparkPlan (26)
: : : +- Project (9)
: : : +- Filter (8)
: : : +- InMemoryTableScan (5)
: : : +- InMemoryRelation (6)
: : : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: : +- Sort (48)
: : +- Exchange (47)
: : +- Project (46)
: : +- BroadcastHashJoin Inner BuildLeft (45)
: : :- BroadcastExchange (39)
: : : +- Filter (38)
: : : +- InMemoryTableScan (35)
: : : +- InMemoryRelation (36)
: : : +- AdaptiveSparkPlan (37)
: : : +- Project (9)
: : : +- Filter (8)
: : : +- InMemoryTableScan (5)
: : : +- InMemoryRelation (6)
: : : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: : +- Project (44)
: : +- Filter (43)
: : +- InMemoryTableScan (40)
: : +- InMemoryRelation (41)
: : +- AdaptiveSparkPlan (42)
: : +- Filter (18)
: : +- InMemoryTableScan (15)
: : +- InMemoryRelation (16)
: : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (17)
: +- Project (57)
: +- BatchEvalPython (56)
: +- Filter (55)
: +- InMemoryTableScan (52)
: +- InMemoryRelation (53)
: +- AdaptiveSparkPlan (54)
: +- Filter (18)
: +- InMemoryTableScan (15)
: +- InMemoryRelation (16)
: +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (17)
+- Project (144)
+- BroadcastHashJoin LeftAnti BuildRight (143)
:- SortMergeJoin LeftAnti (92)
: :- Sort (77)
: : +- Exchange (76)
: : +- InMemoryTableScan (73)
: : +- InMemoryRelation (74)
: : +- AdaptiveSparkPlan (75)
: : +- Project (9)
: : +- Filter (8)
: : +- InMemoryTableScan (5)
: : +- InMemoryRelation (6)
: : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: +- Sort (91)
: +- Exchange (90)
: +- Project (89)
: +- BroadcastHashJoin Inner BuildLeft (88)
: :- BroadcastExchange (82)
: : +- Filter (81)
: : +- InMemoryTableScan (78)
: : +- InMemoryRelation (79)
: : +- AdaptiveSparkPlan (80)
: : +- Project (9)
: : +- Filter (8)
: : +- InMemoryTableScan (5)
: : +- InMemoryRelation (6)
: : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: +- Project (87)
: +- Filter (86)
: +- InMemoryTableScan (83)
: +- InMemoryRelation (84)
: +- AdaptiveSparkPlan (85)
: +- Filter (18)
: +- InMemoryTableScan (15)
: +- InMemoryRelation (16)
: +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (17)
+- BroadcastExchange (142)
+- Project (141)
+- Filter (140)
+- Window (139)
+- WindowGroupLimit (138)
+- Sort (137)
+- Exchange (136)
+- WindowGroupLimit (135)
+- Sort (134)
+- Project (133)
+- BatchEvalPython (132)
+- Project (131)
+- Filter (130)
+- BatchEvalPython (129)
+- Project (128)
+- BroadcastHashJoin Inner BuildLeft (127)
:- BroadcastExchange (120)
: +- Project (119)
: +- SortMergeJoin LeftAnti (118)
: :- Sort (103)
: : +- Exchange (102)
: : +- Project (101)
: : +- BatchEvalPython (100)
: : +- Project (99)
: : +- Filter (98)
: : +- BatchEvalPython (97)
: : +- Filter (96)
: : +- InMemoryTableScan (93)
: : +- InMemoryRelation (94)
: : +- AdaptiveSparkPlan (95)
: : +- Project (9)
: : +- Filter (8)
: : +- InMemoryTableScan (5)
: : +- InMemoryRelation (6)
: : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: +- Sort (117)
: +- Exchange (116)
: +- Project (115)
: +- BroadcastHashJoin Inner BuildLeft (114)
: :- BroadcastExchange (108)
: : +- Filter (107)
: : +- InMemoryTableScan (104)
: : +- InMemoryRelation (105)
: : +- AdaptiveSparkPlan (106)
: : +- Project (9)
: : +- Filter (8)
: : +- InMemoryTableScan (5)
: : +- InMemoryRelation (6)
: : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: +- Project (113)
: +- Filter (112)
: +- InMemoryTableScan (109)
: +- InMemoryRelation (110)
: +- AdaptiveSparkPlan (111)
: +- Filter (18)
: +- InMemoryTableScan (15)
: +- InMemoryRelation (16)
: +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (17)
+- Project (126)
+- BatchEvalPython (125)
+- Filter (124)
+- InMemoryTableScan (121)
+- InMemoryRelation (122)
+- AdaptiveSparkPlan (123)
+- Filter (18)
+- InMemoryTableScan (15)
+- InMemoryRelation (16)
+- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (17)
(1) InMemoryTableScan
Output: []
(2) InMemoryRelation
Arguments: [brand_context_id#14936228, supplier_id#14936229, declaring_supplier_id#14936235, supplier_type#14936234, canonical_id#14938185, is_primary#14938249, configured_threshold#14938250, match_percentage#14938251, confidence_score#14938252, match_reason#14938253], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@730ae8bf,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
Union
:- *(26) Project [brand_context_id#14936228, supplier_id#14936229, declaring_supplier_id#14936235, supplier_type#14936234, canonical_id#14938185, 0 AS is_primary#14938249, threshold#14936238 AS configured_threshold#14938250, 100.0 AS match_percentage#14938251, 1.0 AS confidence_score#14938252, exact_id_match AS match_reason#14938253]
: +- *(26) BroadcastHashJoin [brand_context_id#14936228, supplier_id#14936229], [brand_context_id#14938177, supplier_id#14938178], Inner, BuildLeft, false
: :- BroadcastQueryStage 13
: : +- BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, false], input[1, string, false]),false), [plan_id=4292567]
: : +- *(1) Filter (isnotnull(brand_context_id#14936228) AND isnotnull(supplier_id#14936229))
: : +- TableCacheQueryStage 0
: : +- InMemoryTableScan [brand_context_id#14936228, supplier_id#14936229, supplier_type#14936234, declaring_supplier_id#14936235, threshold#14936238], [isnotnull(brand_context_id#14936228), isnotnull(supplier_id#14936229)]
: : +- InMemoryRelation [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, threshold#14936238, create_date#14936239], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- AdaptiveSparkPlan isFinalPlan=false
: : +- Project [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, threshold#14936238, create_date#14936239]
: : +- Filter isnull(canonical_id#14936236)
: : +- InMemoryTableScan [brand_context_id#14936228, canonical_id#14936236, city#14936231, country#14936233, create_date#14936239, declaring_supplier_id#14936235, entity_name#14936230, state#14936232, supplier_id#14936229, supplier_type#14936234, threshold#14936238], [isnull(canonical_id#14936236)]
: : +- InMemoryRelation [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, canonical_id#14936236, is_primary#14936237, threshold#14936238, create_date#14936239], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14936228,supplier_id#14936229,entity_name#14936230,city#14936231,state#14936232,country#14936233,supplier_type#14936234,declaring_supplier_id#14936235,canonical_id#14936236,is_primary#14936237,threshold#14936238,create_date#14936239] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
: +- *(26) Project [brand_context_id#14938177, supplier_id#14938178, canonical_id#14938185]
: +- *(26) Filter ((((isnotnull(is_primary#14938186) AND is_primary#14938186) AND isnotnull(canonical_id#14938185)) AND isnotnull(brand_context_id#14938177)) AND isnotnull(supplier_id#14938178))
: +- TableCacheQueryStage 1
: +- InMemoryTableScan [brand_context_id#14938177, canonical_id#14938185, is_primary#14938186, supplier_id#14938178], [isnotnull(is_primary#14938186), is_primary#14938186, isnotnull(canonical_id#14938185), isnotnull(brand_context_id#14938177), isnotnull(supplier_id#14938178)]
: +- InMemoryRelation [brand_context_id#14938177, supplier_id#14938178, canonical_id#14938185, is_primary#14938186, entity_name#14938179, city#14938180, state#14938181, country#14938182, threshold#14938187, create_date#14938188], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- AdaptiveSparkPlan isFinalPlan=false
: +- Filter isnotnull(canonical_id#14936236)
: +- InMemoryTableScan [brand_context_id#14936228, supplier_id#14936229, canonical_id#14936236, is_primary#14936237, entity_name#14936230, city#14936231, state#14936232, country#14936233, threshold#14936238, create_date#14936239], [isnotnull(canonical_id#14936236)]
: +- InMemoryRelation [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, canonical_id#14936236, is_primary#14936237, threshold#14936238, create_date#14936239], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14936228,supplier_id#14936229,entity_name#14936230,city#14936231,state#14936232,country#14936233,supplier_type#14936234,declaring_supplier_id#14936235,canonical_id#14936236,is_primary#14936237,threshold#14936238,create_date#14936239] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
:- *(28) Project [u_brand_context_id#14938293 AS brand_context_id#14938414, u_supplier_id#14938294 AS supplier_id#14938415, u_declaring_supplier_id#14938295 AS declaring_supplier_id#14938416, u_supplier_type#14938296 AS supplier_type#14938417, e_canonical_id#14938317 AS canonical_id#14938418, 0 AS is_primary#14938419, u_threshold#14938297 AS configured_threshold#14938420, cast((similarity#14938374 * 100.0) as double) AS match_percentage#14938527, cast(similarity#14938374 as double) AS confidence_score#14938528, matched_existing AS match_reason#14938423]
: +- *(28) Filter (rank#14938394 = 1)
: +- Window [row_number() windowspecdefinition(u_brand_context_id#14938293, u_supplier_id#14938294, similarity#14938374 DESC NULLS LAST, e_create_date#14938320 ASC NULLS FIRST, e_supplier_id#14938316 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank#14938394], [u_brand_context_id#14938293, u_supplier_id#14938294], [similarity#14938374 DESC NULLS LAST, e_create_date#14938320 ASC NULLS FIRST, e_supplier_id#14938316 ASC NULLS FIRST]
: +- WindowGroupLimit [u_brand_context_id#14938293, u_supplier_id#14938294], [similarity#14938374 DESC NULLS LAST, e_create_date#14938320 ASC NULLS FIRST, e_supplier_id#14938316 ASC NULLS FIRST], row_number(), 1, Final
: +- *(27) Sort [u_brand_context_id#14938293 ASC NULLS FIRST, u_supplier_id#14938294 ASC NULLS FIRST, similarity#14938374 DESC NULLS LAST, e_create_date#14938320 ASC NULLS FIRST, e_supplier_id#14938316 ASC NULLS FIRST], false, 0
: +- ShuffleQueryStage 25
: +- Exchange hashpartitioning(u_brand_context_id#14938293, u_supplier_id#14938294, 10), ENSURE_REQUIREMENTS, [plan_id=4295716]
: +- WindowGroupLimit [u_brand_context_id#14938293, u_supplier_id#14938294], [similarity#14938374 DESC NULLS LAST, e_create_date#14938320 ASC NULLS FIRST, e_supplier_id#14938316 ASC NULLS FIRST], row_number(), 1, Partial
: +- *(19) Sort [u_brand_context_id#14938293 ASC NULLS FIRST, u_supplier_id#14938294 ASC NULLS FIRST, similarity#14938374 DESC NULLS LAST, e_create_date#14938320 ASC NULLS FIRST, e_supplier_id#14938316 ASC NULLS FIRST], false, 0
: +- *(19) Project [u_brand_context_id#14938293, u_supplier_id#14938294, u_declaring_supplier_id#14938295, u_supplier_type#14938296, u_threshold#14938297, e_supplier_id#14938316, e_canonical_id#14938317, e_create_date#14938320, pythonUDF0#14939909 AS similarity#14938374]
: +- BatchEvalPython [calculate_similarity(u_normalized_name#14938298, e_normalized_name#14938318)#14938373], [pythonUDF0#14939909]
: +- *(18) Project [u_brand_context_id#14938293, u_supplier_id#14938294, u_declaring_supplier_id#14938295, u_supplier_type#14938296, u_threshold#14938297, u_normalized_name#14938298, e_supplier_id#14938316, e_canonical_id#14938317, e_normalized_name#14938318, e_create_date#14938320]
: +- *(18) Filter (cast(pythonUDF0#14939908 as double) >= u_threshold#14938297)
: +- BatchEvalPython [calculate_similarity(u_normalized_name#14938298, e_normalized_name#14938318)#14938373], [pythonUDF0#14939908]
: +- *(17) Project [u_brand_context_id#14938293, u_supplier_id#14938294, u_declaring_supplier_id#14938295, u_supplier_type#14938296, u_threshold#14938297, u_normalized_name#14938298, e_supplier_id#14938316, e_canonical_id#14938317, e_normalized_name#14938318, e_create_date#14938320]
: +- *(17) BroadcastHashJoin [u_brand_context_id#14938293, u_location_key#14938302], [e_brand_context_id#14938315, e_location_key#14938319], Inner, BuildLeft, false
: :- BroadcastQueryStage 23
: : +- BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, true], input[6, string, false]),false), [plan_id=4295275]
: : +- *(14) Project [brand_context_id#14938503 AS u_brand_context_id#14938293, supplier_id#14938504 AS u_supplier_id#14938294, declaring_supplier_id#14938510 AS u_declaring_supplier_id#14938295, supplier_type#14938509 AS u_supplier_type#14938296, threshold#14938513 AS u_threshold#14938297, normalized_name#14937760 AS u_normalized_name#14938298, location_key#14937817 AS u_location_key#14938302]
: : +- AQEShuffleRead coalesced
: : +- ShuffleQueryStage 14
: : +- Exchange hashpartitioning(brand_context_id#14938503, supplier_id#14938504, 10), ENSURE_REQUIREMENTS, [plan_id=4292649]
: : +- *(4) Project [brand_context_id#14938503, supplier_id#14938504, supplier_type#14938509, declaring_supplier_id#14938510, threshold#14938513, pythonUDF0#14939900 AS normalized_name#14937760, concat_ws(|, coalesce(pythonUDF1#14939901, ), coalesce(pythonUDF2#14939902, ), coalesce(pythonUDF3#14939903, )) AS location_key#14937817]
: : +- BatchEvalPython [normalize_name(entity_name#14938505)#14937759, normalize_location(city#14938506)#14937772, normalize_location(state#14938507)#14937786, normalize_location(country#14938508)#14937801], [pythonUDF0#14939900, pythonUDF1#14939901, pythonUDF2#14939902, pythonUDF3#14939903]
: : +- *(3) Project [brand_context_id#14938503, supplier_id#14938504, entity_name#14938505, city#14938506, state#14938507, country#14938508, supplier_type#14938509, declaring_supplier_id#14938510, threshold#14938513]
: : +- *(3) Filter ((isnotnull(pythonUDF0#14939897) AND isnotnull(pythonUDF1#14939898)) AND isnotnull(pythonUDF2#14939899))
: : +- BatchEvalPython [normalize_location(city#14938506)#14937772, normalize_location(state#14938507)#14937786, normalize_location(country#14938508)#14937801], [pythonUDF0#14939897, pythonUDF1#14939898, pythonUDF2#14939899]
: : +- *(2) Filter (isnotnull(threshold#14938513) AND isnotnull(brand_context_id#14938503))
: : +- TableCacheQueryStage 2
: : +- InMemoryTableScan [brand_context_id#14938503, supplier_id#14938504, entity_name#14938505, city#14938506, state#14938507, country#14938508, supplier_type#14938509, declaring_supplier_id#14938510, threshold#14938513], [isnotnull(threshold#14938513), isnotnull(brand_context_id#14938503)]
: : +- InMemoryRelation [brand_context_id#14938503, supplier_id#14938504, entity_name#14938505, city#14938506, state#14938507, country#14938508, supplier_type#14938509, declaring_supplier_id#14938510, threshold#14938513, create_date#14938514], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- AdaptiveSparkPlan isFinalPlan=false
: : +- Project [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, threshold#14936238, create_date#14936239]
: : +- Filter isnull(canonical_id#14936236)
: : +- InMemoryTableScan [brand_context_id#14936228, canonical_id#14936236, city#14936231, country#14936233, create_date#14936239, declaring_supplier_id#14936235, entity_name#14936230, state#14936232, supplier_id#14936229, supplier_type#14936234, threshold#14936238], [isnull(canonical_id#14936236)]
: : +- InMemoryRelation [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, canonical_id#14936236, is_primary#14936237, threshold#14936238, create_date#14936239], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14936228,supplier_id#14936229,entity_name#14936230,city#14936231,state#14936232,country#14936233,supplier_type#14936234,declaring_supplier_id#14936235,canonical_id#14936236,is_primary#14936237,threshold#14936238,create_date#14936239] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
: +- *(17) Project [brand_context_id#14938327 AS e_brand_context_id#14938315, supplier_id#14938328 AS e_supplier_id#14938316, canonical_id#14938335 AS e_canonical_id#14938317, pythonUDF0#14939904 AS e_normalized_name#14938318, concat_ws(|, coalesce(pythonUDF1#14939905, ), coalesce(pythonUDF2#14939906, ), coalesce(pythonUDF3#14939907, )) AS e_location_key#14938319, create_date#14938338 AS e_create_date#14938320]
: +- BatchEvalPython [normalize_name(entity_name#14938329)#14937833, normalize_location(city#14938330)#14937846, normalize_location(state#14938331)#14937860, normalize_location(country#14938332)#14937875], [pythonUDF0#14939904, pythonUDF1#14939905, pythonUDF2#14939906, pythonUDF3#14939907]
: +- *(16) Filter isnotnull(brand_context_id#14938327)
: +- TableCacheQueryStage 5
: +- InMemoryTableScan [brand_context_id#14938327, supplier_id#14938328, canonical_id#14938335, entity_name#14938329, city#14938330, state#14938331, country#14938332, create_date#14938338], [isnotnull(brand_context_id#14938327)]
: +- InMemoryRelation [brand_context_id#14938327, supplier_id#14938328, canonical_id#14938335, is_primary#14938336, entity_name#14938329, city#14938330, state#14938331, country#14938332, threshold#14938337, create_date#14938338], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- AdaptiveSparkPlan isFinalPlan=false
: +- Filter isnotnull(canonical_id#14936236)
: +- InMemoryTableScan [brand_context_id#14936228, supplier_id#14936229, canonical_id#14936236, is_primary#14936237, entity_name#14936230, city#14936231, state#14936232, country#14936233, threshold#14936238, create_date#14936239], [isnotnull(canonical_id#14936236)]
: +- InMemoryRelation [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, canonical_id#14936236, is_primary#14936237, threshold#14936238, create_date#14936239], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14936228,supplier_id#14936229,entity_name#14936230,city#14936231,state#14936232,country#14936233,supplier_type#14936234,declaring_supplier_id#14936235,canonical_id#14936236,is_primary#14936237,threshold#14936238,create_date#14936239] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- *(29) Project [brand_context_id#14938539, supplier_id#14938540, declaring_supplier_id#14938546, supplier_type#14938545, supplier_id#14938540 AS canonical_id#14938487, 1 AS is_primary#14938488, threshold#14938549 AS configured_threshold#14938489, 0.0 AS match_percentage#14938490, 0.0 AS confidence_score#14938491, new_canonical AS match_reason#14938492]
+- ShuffleQueryStage 16
+- Exchange hashpartitioning(brand_context_id#14938539, supplier_id#14938540, 10), ENSURE_REQUIREMENTS, [plan_id=4292148]
+- TableCacheQueryStage 6
+- InMemoryTableScan [brand_context_id#14938539, supplier_id#14938540, supplier_type#14938545, declaring_supplier_id#14938546, threshold#14938549]
+- InMemoryRelation [brand_context_id#14938539, supplier_id#14938540, entity_name#14938541, city#14938542, state#14938543, country#14938544, supplier_type#14938545, declaring_supplier_id#14938546, threshold#14938549, create_date#14938550], StorageLevel(disk, memory, deserialized, 1 replicas)
+- AdaptiveSparkPlan isFinalPlan=false
+- Project [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, threshold#14936238, create_date#14936239]
+- Filter isnull(canonical_id#14936236)
+- InMemoryTableScan [brand_context_id#14936228, canonical_id#14936236, city#14936231, country#14936233, create_date#14936239, declaring_supplier_id#14936235, entity_name#14936230, state#14936232, supplier_id#14936229, supplier_type#14936234, threshold#14936238], [isnull(canonical_id#14936236)]
+- InMemoryRelation [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, canonical_id#14936236, is_primary#14936237, threshold#14936238, create_date#14936239], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14936228,supplier_id#14936229,entity_name#14936230,city#14936231,state#14936232,country#14936233,supplier_type#14936234,declaring_supplier_id#14936235,canonical_id#14936236,is_primary#14936237,threshold#14936238,create_date#14936239] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Union
:- Project [brand_context_id#14936228, supplier_id#14936229, declaring_supplier_id#14936235, supplier_type#14936234, canonical_id#14938185, 0 AS is_primary#14938249, threshold#14936238 AS configured_threshold#14938250, 100.0 AS match_percentage#14938251, 1.0 AS confidence_score#14938252, exact_id_match AS match_reason#14938253]
: +- BroadcastHashJoin [brand_context_id#14936228, supplier_id#14936229], [brand_context_id#14938177, supplier_id#14938178], Inner, BuildLeft, false
: :- BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, false], input[1, string, false]),false), [plan_id=4291834]
: : +- Filter (isnotnull(brand_context_id#14936228) AND isnotnull(supplier_id#14936229))
: : +- InMemoryTableScan [brand_context_id#14936228, supplier_id#14936229, supplier_type#14936234, declaring_supplier_id#14936235, threshold#14936238], [isnotnull(brand_context_id#14936228), isnotnull(supplier_id#14936229)]
: : +- InMemoryRelation [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, threshold#14936238, create_date#14936239], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- AdaptiveSparkPlan isFinalPlan=false
: : +- Project [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, threshold#14936238, create_date#14936239]
: : +- Filter isnull(canonical_id#14936236)
: : +- InMemoryTableScan [brand_context_id#14936228, canonical_id#14936236, city#14936231, country#14936233, create_date#14936239, declaring_supplier_id#14936235, entity_name#14936230, state#14936232, supplier_id#14936229, supplier_type#14936234, threshold#14936238], [isnull(canonical_id#14936236)]
: : +- InMemoryRelation [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, canonical_id#14936236, is_primary#14936237, threshold#14936238, create_date#14936239], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14936228,supplier_id#14936229,entity_name#14936230,city#14936231,state#14936232,country#14936233,supplier_type#14936234,declaring_supplier_id#14936235,canonical_id#14936236,is_primary#14936237,threshold#14936238,create_date#14936239] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
: +- Project [brand_context_id#14938177, supplier_id#14938178, canonical_id#14938185]
: +- Filter ((((isnotnull(is_primary#14938186) AND is_primary#14938186) AND isnotnull(canonical_id#14938185)) AND isnotnull(brand_context_id#14938177)) AND isnotnull(supplier_id#14938178))
: +- InMemoryTableScan [brand_context_id#14938177, canonical_id#14938185, is_primary#14938186, supplier_id#14938178], [isnotnull(is_primary#14938186), is_primary#14938186, isnotnull(canonical_id#14938185), isnotnull(brand_context_id#14938177), isnotnull(supplier_id#14938178)]
: +- InMemoryRelation [brand_context_id#14938177, supplier_id#14938178, canonical_id#14938185, is_primary#14938186, entity_name#14938179, city#14938180, state#14938181, country#14938182, threshold#14938187, create_date#14938188], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- AdaptiveSparkPlan isFinalPlan=false
: +- Filter isnotnull(canonical_id#14936236)
: +- InMemoryTableScan [brand_context_id#14936228, supplier_id#14936229, canonical_id#14936236, is_primary#14936237, entity_name#14936230, city#14936231, state#14936232, country#14936233, threshold#14936238, create_date#14936239], [isnotnull(canonical_id#14936236)]
: +- InMemoryRelation [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, canonical_id#14936236, is_primary#14936237, threshold#14936238, create_date#14936239], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14936228,supplier_id#14936229,entity_name#14936230,city#14936231,state#14936232,country#14936233,supplier_type#14936234,declaring_supplier_id#14936235,canonical_id#14936236,is_primary#14936237,threshold#14936238,create_date#14936239] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
:- Project [u_brand_context_id#14938293 AS brand_context_id#14938414, u_supplier_id#14938294 AS supplier_id#14938415, u_declaring_supplier_id#14938295 AS declaring_supplier_id#14938416, u_supplier_type#14938296 AS supplier_type#14938417, e_canonical_id#14938317 AS canonical_id#14938418, 0 AS is_primary#14938419, u_threshold#14938297 AS configured_threshold#14938420, cast((similarity#14938374 * 100.0) as double) AS match_percentage#14938527, cast(similarity#14938374 as double) AS confidence_score#14938528, matched_existing AS match_reason#14938423]
: +- Filter (rank#14938394 = 1)
: +- Window [row_number() windowspecdefinition(u_brand_context_id#14938293, u_supplier_id#14938294, similarity#14938374 DESC NULLS LAST, e_create_date#14938320 ASC NULLS FIRST, e_supplier_id#14938316 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank#14938394], [u_brand_context_id#14938293, u_supplier_id#14938294], [similarity#14938374 DESC NULLS LAST, e_create_date#14938320 ASC NULLS FIRST, e_supplier_id#14938316 ASC NULLS FIRST]
: +- WindowGroupLimit [u_brand_context_id#14938293, u_supplier_id#14938294], [similarity#14938374 DESC NULLS LAST, e_create_date#14938320 ASC NULLS FIRST, e_supplier_id#14938316 ASC NULLS FIRST], row_number(), 1, Final
: +- Sort [u_brand_context_id#14938293 ASC NULLS FIRST, u_supplier_id#14938294 ASC NULLS FIRST, similarity#14938374 DESC NULLS LAST, e_create_date#14938320 ASC NULLS FIRST, e_supplier_id#14938316 ASC NULLS FIRST], false, 0
: +- Exchange hashpartitioning(u_brand_context_id#14938293, u_supplier_id#14938294, 10), ENSURE_REQUIREMENTS, [plan_id=4291861]
: +- WindowGroupLimit [u_brand_context_id#14938293, u_supplier_id#14938294], [similarity#14938374 DESC NULLS LAST, e_create_date#14938320 ASC NULLS FIRST, e_supplier_id#14938316 ASC NULLS FIRST], row_number(), 1, Partial
: +- Sort [u_brand_context_id#14938293 ASC NULLS FIRST, u_supplier_id#14938294 ASC NULLS FIRST, similarity#14938374 DESC NULLS LAST, e_create_date#14938320 ASC NULLS FIRST, e_supplier_id#14938316 ASC NULLS FIRST], false, 0
: +- Project [u_brand_context_id#14938293, u_supplier_id#14938294, u_declaring_supplier_id#14938295, u_supplier_type#14938296, u_threshold#14938297, e_supplier_id#14938316, e_canonical_id#14938317, e_create_date#14938320, pythonUDF0#14939909 AS similarity#14938374]
: +- BatchEvalPython [calculate_similarity(u_normalized_name#14938298, e_normalized_name#14938318)#14938373], [pythonUDF0#14939909]
: +- Project [u_brand_context_id#14938293, u_supplier_id#14938294, u_declaring_supplier_id#14938295, u_supplier_type#14938296, u_threshold#14938297, u_normalized_name#14938298, e_supplier_id#14938316, e_canonical_id#14938317, e_normalized_name#14938318, e_create_date#14938320]
: +- Filter (cast(pythonUDF0#14939908 as double) >= u_threshold#14938297)
: +- BatchEvalPython [calculate_similarity(u_normalized_name#14938298, e_normalized_name#14938318)#14938373], [pythonUDF0#14939908]
: +- Project [u_brand_context_id#14938293, u_supplier_id#14938294, u_declaring_supplier_id#14938295, u_supplier_type#14938296, u_threshold#14938297, u_normalized_name#14938298, e_supplier_id#14938316, e_canonical_id#14938317, e_normalized_name#14938318, e_create_date#14938320]
: +- BroadcastHashJoin [u_brand_context_id#14938293, u_location_key#14938302], [e_brand_context_id#14938315, e_location_key#14938319], Inner, BuildLeft, false
: :- BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, true], input[6, string, false]),false), [plan_id=4291849]
: : +- Project [brand_context_id#14938503 AS u_brand_context_id#14938293, supplier_id#14938504 AS u_supplier_id#14938294, declaring_supplier_id#14938510 AS u_declaring_supplier_id#14938295, supplier_type#14938509 AS u_supplier_type#14938296, threshold#14938513 AS u_threshold#14938297, normalized_name#14937760 AS u_normalized_name#14938298, location_key#14937817 AS u_location_key#14938302]
: : +- SortMergeJoin [brand_context_id#14938503, supplier_id#14938504], [brand_context_id#14938266, supplier_id#14938267], LeftAnti
: : :- Sort [brand_context_id#14938503 ASC NULLS FIRST, supplier_id#14938504 ASC NULLS FIRST], false, 0
: : : +- Exchange hashpartitioning(brand_context_id#14938503, supplier_id#14938504, 10), ENSURE_REQUIREMENTS, [plan_id=4291842]
: : : +- Project [brand_context_id#14938503, supplier_id#14938504, supplier_type#14938509, declaring_supplier_id#14938510, threshold#14938513, pythonUDF0#14939900 AS normalized_name#14937760, concat_ws(|, coalesce(pythonUDF1#14939901, ), coalesce(pythonUDF2#14939902, ), coalesce(pythonUDF3#14939903, )) AS location_key#14937817]
: : : +- BatchEvalPython [normalize_name(entity_name#14938505)#14937759, normalize_location(city#14938506)#14937772, normalize_location(state#14938507)#14937786, normalize_location(country#14938508)#14937801], [pythonUDF0#14939900, pythonUDF1#14939901, pythonUDF2#14939902, pythonUDF3#14939903]
: : : +- Project [brand_context_id#14938503, supplier_id#14938504, entity_name#14938505, city#14938506, state#14938507, country#14938508, supplier_type#14938509, declaring_supplier_id#14938510, threshold#14938513]
: : : +- Filter ((isnotnull(pythonUDF0#14939897) AND isnotnull(pythonUDF1#14939898)) AND isnotnull(pythonUDF2#14939899))
: : : +- BatchEvalPython [normalize_location(city#14938506)#14937772, normalize_location(state#14938507)#14937786, normalize_location(country#14938508)#14937801], [pythonUDF0#14939897, pythonUDF1#14939898, pythonUDF2#14939899]
: : : +- Filter (isnotnull(threshold#14938513) AND isnotnull(brand_context_id#14938503))
: : : +- InMemoryTableScan [brand_context_id#14938503, supplier_id#14938504, entity_name#14938505, city#14938506, state#14938507, country#14938508, supplier_type#14938509, declaring_supplier_id#14938510, threshold#14938513], [isnotnull(threshold#14938513), isnotnull(brand_context_id#14938503)]
: : : +- InMemoryRelation [brand_context_id#14938503, supplier_id#14938504, entity_name#14938505, city#14938506, state#14938507, country#14938508, supplier_type#14938509, declaring_supplier_id#14938510, threshold#14938513, create_date#14938514], StorageLevel(disk, memory, deserialized, 1 replicas)
: : : +- AdaptiveSparkPlan isFinalPlan=false
: : : +- Project [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, threshold#14936238, create_date#14936239]
: : : +- Filter isnull(canonical_id#14936236)
: : : +- InMemoryTableScan [brand_context_id#14936228, canonical_id#14936236, city#14936231, country#14936233, create_date#14936239, declaring_supplier_id#14936235, entity_name#14936230, state#14936232, supplier_id#14936229, supplier_type#14936234, threshold#14936238], [isnull(canonical_id#14936236)]
: : : +- InMemoryRelation [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, canonical_id#14936236, is_primary#14936237, threshold#14936238, create_date#14936239], StorageLevel(disk, memory, deserialized, 1 replicas)
: : : +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14936228,supplier_id#14936229,entity_name#14936230,city#14936231,state#14936232,country#14936233,supplier_type#14936234,declaring_supplier_id#14936235,canonical_id#14936236,is_primary#14936237,threshold#14936238,create_date#14936239] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
: : +- Sort [brand_context_id#14938266 ASC NULLS FIRST, supplier_id#14938267 ASC NULLS FIRST], false, 0
: : +- Exchange hashpartitioning(brand_context_id#14938266, supplier_id#14938267, 10), ENSURE_REQUIREMENTS, [plan_id=4291843]
: : +- Project [brand_context_id#14938266, supplier_id#14938267]
: : +- BroadcastHashJoin [brand_context_id#14938266, supplier_id#14938267], [brand_context_id#14938515, supplier_id#14938516], Inner, BuildLeft, false
: : :- BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, false], input[1, string, false]),false), [plan_id=4291837]
: : : +- Filter (isnotnull(brand_context_id#14938266) AND isnotnull(supplier_id#14938267))
: : : +- InMemoryTableScan [brand_context_id#14938266, supplier_id#14938267], [isnotnull(brand_context_id#14938266), isnotnull(supplier_id#14938267)]
: : : +- InMemoryRelation [brand_context_id#14938266, supplier_id#14938267, entity_name#14938268, city#14938269, state#14938270, country#14938271, supplier_type#14938272, declaring_supplier_id#14938273, threshold#14938276, create_date#14938277], StorageLevel(disk, memory, deserialized, 1 replicas)
: : : +- AdaptiveSparkPlan isFinalPlan=false
: : : +- Project [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, threshold#14936238, create_date#14936239]
: : : +- Filter isnull(canonical_id#14936236)
: : : +- InMemoryTableScan [brand_context_id#14936228, canonical_id#14936236, city#14936231, country#14936233, create_date#14936239, declaring_supplier_id#14936235, entity_name#14936230, state#14936232, supplier_id#14936229, supplier_type#14936234, threshold#14936238], [isnull(canonical_id#14936236)]
: : : +- InMemoryRelation [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, canonical_id#14936236, is_primary#14936237, threshold#14936238, create_date#14936239], StorageLevel(disk, memory, deserialized, 1 replicas)
: : : +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14936228,supplier_id#14936229,entity_name#14936230,city#14936231,state#14936232,country#14936233,supplier_type#14936234,declaring_supplier_id#14936235,canonical_id#14936236,is_primary#14936237,threshold#14936238,create_date#14936239] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
: : +- Project [brand_context_id#14938515, supplier_id#14938516]
: : +- Filter ((((isnotnull(is_primary#14938524) AND is_primary#14938524) AND isnotnull(canonical_id#14938523)) AND isnotnull(brand_context_id#14938515)) AND isnotnull(supplier_id#14938516))
: : +- InMemoryTableScan [brand_context_id#14938515, canonical_id#14938523, is_primary#14938524, supplier_id#14938516], [isnotnull(is_primary#14938524), is_primary#14938524, isnotnull(canonical_id#14938523), isnotnull(brand_context_id#14938515), isnotnull(supplier_id#14938516)]
: : +- InMemoryRelation [brand_context_id#14938515, supplier_id#14938516, canonical_id#14938523, is_primary#14938524, entity_name#14938517, city#14938518, state#14938519, country#14938520, threshold#14938525, create_date#14938526], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- AdaptiveSparkPlan isFinalPlan=false
: : +- Filter isnotnull(canonical_id#14936236)
: : +- InMemoryTableScan [brand_context_id#14936228, supplier_id#14936229, canonical_id#14936236, is_primary#14936237, entity_name#14936230, city#14936231, state#14936232, country#14936233, threshold#14936238, create_date#14936239], [isnotnull(canonical_id#14936236)]
: : +- InMemoryRelation [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, canonical_id#14936236, is_primary#14936237, threshold#14936238, create_date#14936239], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14936228,supplier_id#14936229,entity_name#14936230,city#14936231,state#14936232,country#14936233,supplier_type#14936234,declaring_supplier_id#14936235,canonical_id#14936236,is_primary#14936237,threshold#14936238,create_date#14936239] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
: +- Project [brand_context_id#14938327 AS e_brand_context_id#14938315, supplier_id#14938328 AS e_supplier_id#14938316, canonical_id#14938335 AS e_canonical_id#14938317, pythonUDF0#14939904 AS e_normalized_name#14938318, concat_ws(|, coalesce(pythonUDF1#14939905, ), coalesce(pythonUDF2#14939906, ), coalesce(pythonUDF3#14939907, )) AS e_location_key#14938319, create_date#14938338 AS e_create_date#14938320]
: +- BatchEvalPython [normalize_name(entity_name#14938329)#14937833, normalize_location(city#14938330)#14937846, normalize_location(state#14938331)#14937860, normalize_location(country#14938332)#14937875], [pythonUDF0#14939904, pythonUDF1#14939905, pythonUDF2#14939906, pythonUDF3#14939907]
: +- Filter isnotnull(brand_context_id#14938327)
: +- InMemoryTableScan [brand_context_id#14938327, supplier_id#14938328, canonical_id#14938335, entity_name#14938329, city#14938330, state#14938331, country#14938332, create_date#14938338], [isnotnull(brand_context_id#14938327)]
: +- InMemoryRelation [brand_context_id#14938327, supplier_id#14938328, canonical_id#14938335, is_primary#14938336, entity_name#14938329, city#14938330, state#14938331, country#14938332, threshold#14938337, create_date#14938338], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- AdaptiveSparkPlan isFinalPlan=false
: +- Filter isnotnull(canonical_id#14936236)
: +- InMemoryTableScan [brand_context_id#14936228, supplier_id#14936229, canonical_id#14936236, is_primary#14936237, entity_name#14936230, city#14936231, state#14936232, country#14936233, threshold#14936238, create_date#14936239], [isnotnull(canonical_id#14936236)]
: +- InMemoryRelation [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, canonical_id#14936236, is_primary#14936237, threshold#14936238, create_date#14936239], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14936228,supplier_id#14936229,entity_name#14936230,city#14936231,state#14936232,country#14936233,supplier_type#14936234,declaring_supplier_id#14936235,canonical_id#14936236,is_primary#14936237,threshold#14936238,create_date#14936239] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- Project [brand_context_id#14938539, supplier_id#14938540, declaring_supplier_id#14938546, supplier_type#14938545, supplier_id#14938540 AS canonical_id#14938487, 1 AS is_primary#14938488, threshold#14938549 AS configured_threshold#14938489, 0.0 AS match_percentage#14938490, 0.0 AS confidence_score#14938491, new_canonical AS match_reason#14938492]
+- BroadcastHashJoin [brand_context_id#14938539, supplier_id#14938540], [brand_context_id#14938414, supplier_id#14938415], LeftAnti, BuildRight, false
:- SortMergeJoin [brand_context_id#14938539, supplier_id#14938540], [brand_context_id#14938551, supplier_id#14938552], LeftAnti
: :- Sort [brand_context_id#14938539 ASC NULLS FIRST, supplier_id#14938540 ASC NULLS FIRST], false, 0
: : +- Exchange hashpartitioning(brand_context_id#14938539, supplier_id#14938540, 10), ENSURE_REQUIREMENTS, [plan_id=4291872]
: : +- InMemoryTableScan [brand_context_id#14938539, supplier_id#14938540, supplier_type#14938545, declaring_supplier_id#14938546, threshold#14938549]
: : +- InMemoryRelation [brand_context_id#14938539, supplier_id#14938540, entity_name#14938541, city#14938542, state#14938543, country#14938544, supplier_type#14938545, declaring_supplier_id#14938546, threshold#14938549, create_date#14938550], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- AdaptiveSparkPlan isFinalPlan=false
: : +- Project [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, threshold#14936238, create_date#14936239]
: : +- Filter isnull(canonical_id#14936236)
: : +- InMemoryTableScan [brand_context_id#14936228, canonical_id#14936236, city#14936231, country#14936233, create_date#14936239, declaring_supplier_id#14936235, entity_name#14936230, state#14936232, supplier_id#14936229, supplier_type#14936234, threshold#14936238], [isnull(canonical_id#14936236)]
: : +- InMemoryRelation [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, canonical_id#14936236, is_primary#14936237, threshold#14936238, create_date#14936239], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14936228,supplier_id#14936229,entity_name#14936230,city#14936231,state#14936232,country#14936233,supplier_type#14936234,declaring_supplier_id#14936235,canonical_id#14936236,is_primary#14936237,threshold#14936238,create_date#14936239] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
: +- Sort [brand_context_id#14938551 ASC NULLS FIRST, supplier_id#14938552 ASC NULLS FIRST], false, 0
: +- Exchange hashpartitioning(brand_context_id#14938551, supplier_id#14938552, 10), ENSURE_REQUIREMENTS, [plan_id=4291873]
: +- Project [brand_context_id#14938551, supplier_id#14938552]
: +- BroadcastHashJoin [brand_context_id#14938551, supplier_id#14938552], [brand_context_id#14938563, supplier_id#14938564], Inner, BuildLeft, false
: :- BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, false], input[1, string, false]),false), [plan_id=4291867]
: : +- Filter (isnotnull(brand_context_id#14938551) AND isnotnull(supplier_id#14938552))
: : +- InMemoryTableScan [brand_context_id#14938551, supplier_id#14938552], [isnotnull(brand_context_id#14938551), isnotnull(supplier_id#14938552)]
: : +- InMemoryRelation [brand_context_id#14938551, supplier_id#14938552, entity_name#14938553, city#14938554, state#14938555, country#14938556, supplier_type#14938557, declaring_supplier_id#14938558, threshold#14938561, create_date#14938562], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- AdaptiveSparkPlan isFinalPlan=false
: : +- Project [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, threshold#14936238, create_date#14936239]
: : +- Filter isnull(canonical_id#14936236)
: : +- InMemoryTableScan [brand_context_id#14936228, canonical_id#14936236, city#14936231, country#14936233, create_date#14936239, declaring_supplier_id#14936235, entity_name#14936230, state#14936232, supplier_id#14936229, supplier_type#14936234, threshold#14936238], [isnull(canonical_id#14936236)]
: : +- InMemoryRelation [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, canonical_id#14936236, is_primary#14936237, threshold#14936238, create_date#14936239], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14936228,supplier_id#14936229,entity_name#14936230,city#14936231,state#14936232,country#14936233,supplier_type#14936234,declaring_supplier_id#14936235,canonical_id#14936236,is_primary#14936237,threshold#14936238,create_date#14936239] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
: +- Project [brand_context_id#14938563, supplier_id#14938564]
: +- Filter ((((isnotnull(is_primary#14938572) AND is_primary#14938572) AND isnotnull(canonical_id#14938571)) AND isnotnull(brand_context_id#14938563)) AND isnotnull(supplier_id#14938564))
: +- InMemoryTableScan [brand_context_id#14938563, canonical_id#14938571, is_primary#14938572, supplier_id#14938564], [isnotnull(is_primary#14938572), is_primary#14938572, isnotnull(canonical_id#14938571), isnotnull(brand_context_id#14938563), isnotnull(supplier_id#14938564)]
: +- InMemoryRelation [brand_context_id#14938563, supplier_id#14938564, canonical_id#14938571, is_primary#14938572, entity_name#14938565, city#14938566, state#14938567, country#14938568, threshold#14938573, create_date#14938574], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- AdaptiveSparkPlan isFinalPlan=false
: +- Filter isnotnull(canonical_id#14936236)
: +- InMemoryTableScan [brand_context_id#14936228, supplier_id#14936229, canonical_id#14936236, is_primary#14936237, entity_name#14936230, city#14936231, state#14936232, country#14936233, threshold#14936238, create_date#14936239], [isnotnull(canonical_id#14936236)]
: +- InMemoryRelation [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, canonical_id#14936236, is_primary#14936237, threshold#14936238, create_date#14936239], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14936228,supplier_id#14936229,entity_name#14936230,city#14936231,state#14936232,country#14936233,supplier_type#14936234,declaring_supplier_id#14936235,canonical_id#14936236,is_primary#14936237,threshold#14936238,create_date#14936239] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true]),false), [plan_id=4291908]
+- Project [u_brand_context_id#14938293 AS brand_context_id#14938414, u_supplier_id#14938294 AS supplier_id#14938415]
+- Filter (rank#14938394 = 1)
+- Window [row_number() windowspecdefinition(u_brand_context_id#14938293, u_supplier_id#14938294, similarity#14938374 DESC NULLS LAST, e_create_date#14938320 ASC NULLS FIRST, e_supplier_id#14938316 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank#14938394], [u_brand_context_id#14938293, u_supplier_id#14938294], [similarity#14938374 DESC NULLS LAST, e_create_date#14938320 ASC NULLS FIRST, e_supplier_id#14938316 ASC NULLS FIRST]
+- WindowGroupLimit [u_brand_context_id#14938293, u_supplier_id#14938294], [similarity#14938374 DESC NULLS LAST, e_create_date#14938320 ASC NULLS FIRST, e_supplier_id#14938316 ASC NULLS FIRST], row_number(), 1, Final
+- Sort [u_brand_context_id#14938293 ASC NULLS FIRST, u_supplier_id#14938294 ASC NULLS FIRST, similarity#14938374 DESC NULLS LAST, e_create_date#14938320 ASC NULLS FIRST, e_supplier_id#14938316 ASC NULLS FIRST], false, 0
+- Exchange hashpartitioning(u_brand_context_id#14938293, u_supplier_id#14938294, 10), ENSURE_REQUIREMENTS, [plan_id=4291901]
+- WindowGroupLimit [u_brand_context_id#14938293, u_supplier_id#14938294], [similarity#14938374 DESC NULLS LAST, e_create_date#14938320 ASC NULLS FIRST, e_supplier_id#14938316 ASC NULLS FIRST], row_number(), 1, Partial
+- Sort [u_brand_context_id#14938293 ASC NULLS FIRST, u_supplier_id#14938294 ASC NULLS FIRST, similarity#14938374 DESC NULLS LAST, e_create_date#14938320 ASC NULLS FIRST, e_supplier_id#14938316 ASC NULLS FIRST], false, 0
+- Project [u_brand_context_id#14938293, u_supplier_id#14938294, e_supplier_id#14938316, e_create_date#14938320, pythonUDF0#14939922 AS similarity#14938374]
+- BatchEvalPython [calculate_similarity(u_normalized_name#14938298, e_normalized_name#14938318)#14938373], [pythonUDF0#14939922]
+- Project [u_brand_context_id#14938293, u_supplier_id#14938294, u_normalized_name#14938298, e_supplier_id#14938316, e_normalized_name#14938318, e_create_date#14938320]
+- Filter (cast(pythonUDF0#14939921 as double) >= u_threshold#14938297)
+- BatchEvalPython [calculate_similarity(u_normalized_name#14938298, e_normalized_name#14938318)#14938373], [pythonUDF0#14939921]
+- Project [u_brand_context_id#14938293, u_supplier_id#14938294, u_threshold#14938297, u_normalized_name#14938298, e_supplier_id#14938316, e_normalized_name#14938318, e_create_date#14938320]
+- BroadcastHashJoin [u_brand_context_id#14938293, u_location_key#14938302], [e_brand_context_id#14938315, e_location_key#14938319], Inner, BuildLeft, false
:- BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, true], input[4, string, false]),false), [plan_id=4291889]
: +- Project [brand_context_id#14938436 AS u_brand_context_id#14938293, supplier_id#14938437 AS u_supplier_id#14938294, threshold#14938446 AS u_threshold#14938297, normalized_name#14937760 AS u_normalized_name#14938298, location_key#14937817 AS u_location_key#14938302]
: +- SortMergeJoin [brand_context_id#14938436, supplier_id#14938437], [brand_context_id#14938448, supplier_id#14938449], LeftAnti
: :- Sort [brand_context_id#14938436 ASC NULLS FIRST, supplier_id#14938437 ASC NULLS FIRST], false, 0
: : +- Exchange hashpartitioning(brand_context_id#14938436, supplier_id#14938437, 10), ENSURE_REQUIREMENTS, [plan_id=4291882]
: : +- Project [brand_context_id#14938436, supplier_id#14938437, threshold#14938446, pythonUDF0#14939913 AS normalized_name#14937760, concat_ws(|, coalesce(pythonUDF1#14939914, ), coalesce(pythonUDF2#14939915, ), coalesce(pythonUDF3#14939916, )) AS location_key#14937817]
: : +- BatchEvalPython [normalize_name(entity_name#14938438)#14937759, normalize_location(city#14938439)#14937772, normalize_location(state#14938440)#14937786, normalize_location(country#14938441)#14937801], [pythonUDF0#14939913, pythonUDF1#14939914, pythonUDF2#14939915, pythonUDF3#14939916]
: : +- Project [brand_context_id#14938436, supplier_id#14938437, entity_name#14938438, city#14938439, state#14938440, country#14938441, threshold#14938446]
: : +- Filter ((isnotnull(pythonUDF0#14939910) AND isnotnull(pythonUDF1#14939911)) AND isnotnull(pythonUDF2#14939912))
: : +- BatchEvalPython [normalize_location(city#14938439)#14937772, normalize_location(state#14938440)#14937786, normalize_location(country#14938441)#14937801], [pythonUDF0#14939910, pythonUDF1#14939911, pythonUDF2#14939912]
: : +- Filter ((isnotnull(threshold#14938446) AND isnotnull(brand_context_id#14938436)) AND isnotnull(supplier_id#14938437))
: : +- InMemoryTableScan [brand_context_id#14938436, supplier_id#14938437, entity_name#14938438, city#14938439, state#14938440, country#14938441, threshold#14938446], [isnotnull(threshold#14938446), isnotnull(brand_context_id#14938436), isnotnull(supplier_id#14938437)]
: : +- InMemoryRelation [brand_context_id#14938436, supplier_id#14938437, entity_name#14938438, city#14938439, state#14938440, country#14938441, supplier_type#14938442, declaring_supplier_id#14938443, threshold#14938446, create_date#14938447], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- AdaptiveSparkPlan isFinalPlan=false
: : +- Project [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, threshold#14936238, create_date#14936239]
: : +- Filter isnull(canonical_id#14936236)
: : +- InMemoryTableScan [brand_context_id#14936228, canonical_id#14936236, city#14936231, country#14936233, create_date#14936239, declaring_supplier_id#14936235, entity_name#14936230, state#14936232, supplier_id#14936229, supplier_type#14936234, threshold#14936238], [isnull(canonical_id#14936236)]
: : +- InMemoryRelation [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, canonical_id#14936236, is_primary#14936237, threshold#14936238, create_date#14936239], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14936228,supplier_id#14936229,entity_name#14936230,city#14936231,state#14936232,country#14936233,supplier_type#14936234,declaring_supplier_id#14936235,canonical_id#14936236,is_primary#14936237,threshold#14936238,create_date#14936239] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
: +- Sort [brand_context_id#14938448 ASC NULLS FIRST, supplier_id#14938449 ASC NULLS FIRST], false, 0
: +- Exchange hashpartitioning(brand_context_id#14938448, supplier_id#14938449, 10), ENSURE_REQUIREMENTS, [plan_id=4291883]
: +- Project [brand_context_id#14938448, supplier_id#14938449]
: +- BroadcastHashJoin [brand_context_id#14938448, supplier_id#14938449], [brand_context_id#14938460, supplier_id#14938461], Inner, BuildLeft, false
: :- BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, false], input[1, string, false]),false), [plan_id=4291877]
: : +- Filter (isnotnull(brand_context_id#14938448) AND isnotnull(supplier_id#14938449))
: : +- InMemoryTableScan [brand_context_id#14938448, supplier_id#14938449], [isnotnull(brand_context_id#14938448), isnotnull(supplier_id#14938449)]
: : +- InMemoryRelation [brand_context_id#14938448, supplier_id#14938449, entity_name#14938450, city#14938451, state#14938452, country#14938453, supplier_type#14938454, declaring_supplier_id#14938455, threshold#14938458, create_date#14938459], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- AdaptiveSparkPlan isFinalPlan=false
: : +- Project [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, threshold#14936238, create_date#14936239]
: : +- Filter isnull(canonical_id#14936236)
: : +- InMemoryTableScan [brand_context_id#14936228, canonical_id#14936236, city#14936231, country#14936233, create_date#14936239, declaring_supplier_id#14936235, entity_name#14936230, state#14936232, supplier_id#14936229, supplier_type#14936234, threshold#14936238], [isnull(canonical_id#14936236)]
: : +- InMemoryRelation [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, canonical_id#14936236, is_primary#14936237, threshold#14936238, create_date#14936239], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14936228,supplier_id#14936229,entity_name#14936230,city#14936231,state#14936232,country#14936233,supplier_type#14936234,declaring_supplier_id#14936235,canonical_id#14936236,is_primary#14936237,threshold#14936238,create_date#14936239] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
: +- Project [brand_context_id#14938460, supplier_id#14938461]
: +- Filter ((((isnotnull(is_primary#14938469) AND is_primary#14938469) AND isnotnull(canonical_id#14938468)) AND isnotnull(brand_context_id#14938460)) AND isnotnull(supplier_id#14938461))
: +- InMemoryTableScan [brand_context_id#14938460, canonical_id#14938468, is_primary#14938469, supplier_id#14938461], [isnotnull(is_primary#14938469), is_primary#14938469, isnotnull(canonical_id#14938468), isnotnull(brand_context_id#14938460), isnotnull(supplier_id#14938461)]
: +- InMemoryRelation [brand_context_id#14938460, supplier_id#14938461, canonical_id#14938468, is_primary#14938469, entity_name#14938462, city#14938463, state#14938464, country#14938465, threshold#14938470, create_date#14938471], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- AdaptiveSparkPlan isFinalPlan=false
: +- Filter isnotnull(canonical_id#14936236)
: +- InMemoryTableScan [brand_context_id#14936228, supplier_id#14936229, canonical_id#14936236, is_primary#14936237, entity_name#14936230, city#14936231, state#14936232, country#14936233, threshold#14936238, create_date#14936239], [isnotnull(canonical_id#14936236)]
: +- InMemoryRelation [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, canonical_id#14936236, is_primary#14936237, threshold#14936238, create_date#14936239], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14936228,supplier_id#14936229,entity_name#14936230,city#14936231,state#14936232,country#14936233,supplier_type#14936234,declaring_supplier_id#14936235,canonical_id#14936236,is_primary#14936237,threshold#14936238,create_date#14936239] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- Project [brand_context_id#14938575 AS e_brand_context_id#14938315, supplier_id#14938576 AS e_supplier_id#14938316, pythonUDF0#14939917 AS e_normalized_name#14938318, concat_ws(|, coalesce(pythonUDF1#14939918, ), coalesce(pythonUDF2#14939919, ), coalesce(pythonUDF3#14939920, )) AS e_location_key#14938319, create_date#14938586 AS e_create_date#14938320]
+- BatchEvalPython [normalize_name(entity_name#14938577)#14937833, normalize_location(city#14938578)#14937846, normalize_location(state#14938579)#14937860, normalize_location(country#14938580)#14937875], [pythonUDF0#14939917, pythonUDF1#14939918, pythonUDF2#14939919, pythonUDF3#14939920]
+- Filter isnotnull(brand_context_id#14938575)
+- InMemoryTableScan [brand_context_id#14938575, supplier_id#14938576, entity_name#14938577, city#14938578, state#14938579, country#14938580, create_date#14938586], [isnotnull(brand_context_id#14938575)]
+- InMemoryRelation [brand_context_id#14938575, supplier_id#14938576, canonical_id#14938583, is_primary#14938584, entity_name#14938577, city#14938578, state#14938579, country#14938580, threshold#14938585, create_date#14938586], StorageLevel(disk, memory, deserialized, 1 replicas)
+- AdaptiveSparkPlan isFinalPlan=false
+- Filter isnotnull(canonical_id#14936236)
+- InMemoryTableScan [brand_context_id#14936228, supplier_id#14936229, canonical_id#14936236, is_primary#14936237, entity_name#14936230, city#14936231, state#14936232, country#14936233, threshold#14936238, create_date#14936239], [isnotnull(canonical_id#14936236)]
+- InMemoryRelation [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, canonical_id#14936236, is_primary#14936237, threshold#14936238, create_date#14936239], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14936228,supplier_id#14936229,entity_name#14936230,city#14936231,state#14936232,country#14936233,supplier_type#14936234,declaring_supplier_id#14936235,canonical_id#14936236,is_primary#14936237,threshold#14936238,create_date#14936239] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(3) InMemoryTableScan
Output [5]: [brand_context_id#14936228, supplier_id#14936229, supplier_type#14936234, declaring_supplier_id#14936235, threshold#14936238]
Arguments: [brand_context_id#14936228, supplier_id#14936229, supplier_type#14936234, declaring_supplier_id#14936235, threshold#14936238], [isnotnull(brand_context_id#14936228), isnotnull(supplier_id#14936229)]
(4) InMemoryRelation
Arguments: [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, threshold#14936238, create_date#14936239], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@730ae8bf,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Project [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, threshold#14936238, create_date#14936239]
+- *(1) Filter isnull(canonical_id#14936236)
+- TableCacheQueryStage 0
+- InMemoryTableScan [brand_context_id#14936228, canonical_id#14936236, city#14936231, country#14936233, create_date#14936239, declaring_supplier_id#14936235, entity_name#14936230, state#14936232, supplier_id#14936229, supplier_type#14936234, threshold#14936238], [isnull(canonical_id#14936236)]
+- InMemoryRelation [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, canonical_id#14936236, is_primary#14936237, threshold#14936238, create_date#14936239], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14936228,supplier_id#14936229,entity_name#14936230,city#14936231,state#14936232,country#14936233,supplier_type#14936234,declaring_supplier_id#14936235,canonical_id#14936236,is_primary#14936237,threshold#14936238,create_date#14936239] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Project [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, threshold#14936238, create_date#14936239]
+- Filter isnull(canonical_id#14936236)
+- InMemoryTableScan [brand_context_id#14936228, canonical_id#14936236, city#14936231, country#14936233, create_date#14936239, declaring_supplier_id#14936235, entity_name#14936230, state#14936232, supplier_id#14936229, supplier_type#14936234, threshold#14936238], [isnull(canonical_id#14936236)]
+- InMemoryRelation [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, canonical_id#14936236, is_primary#14936237, threshold#14936238, create_date#14936239], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14936228,supplier_id#14936229,entity_name#14936230,city#14936231,state#14936232,country#14936233,supplier_type#14936234,declaring_supplier_id#14936235,canonical_id#14936236,is_primary#14936237,threshold#14936238,create_date#14936239] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(5) InMemoryTableScan
Output [11]: [brand_context_id#14936228, canonical_id#14936236, city#14936231, country#14936233, create_date#14936239, declaring_supplier_id#14936235, entity_name#14936230, state#14936232, supplier_id#14936229, supplier_type#14936234, threshold#14936238]
Arguments: [brand_context_id#14936228, canonical_id#14936236, city#14936231, country#14936233, create_date#14936239, declaring_supplier_id#14936235, entity_name#14936230, state#14936232, supplier_id#14936229, supplier_type#14936234, threshold#14936238], [isnull(canonical_id#14936236)]
(6) InMemoryRelation
Arguments: [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, canonical_id#14936236, is_primary#14936237, threshold#14936238, create_date#14936239], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@730ae8bf,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14936228,supplier_id#14936229,entity_name#14936230,city#14936231,state#14936232,country#14936233,supplier_type#14936234,declaring_supplier_id#14936235,canonical_id#14936236,is_primary#14936237,threshold#14936238,create_date#14936239] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(7) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [codegen id : 1]
Output [12]: [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, canonical_id#14936236, is_primary#14936237, threshold#14936238, create_date#14936239]
ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,country:string,supplier_type:string,declaring_supplier_id:string,canonical_id:string,is_primary:boolean,threshold:double,create_date:date>
(8) Filter
Input [11]: [brand_context_id#14936228, canonical_id#14936236, city#14936231, country#14936233, create_date#14936239, declaring_supplier_id#14936235, entity_name#14936230, state#14936232, supplier_id#14936229, supplier_type#14936234, threshold#14936238]
Condition : isnull(canonical_id#14936236)
(9) Project
Output [10]: [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, threshold#14936238, create_date#14936239]
Input [11]: [brand_context_id#14936228, canonical_id#14936236, city#14936231, country#14936233, create_date#14936239, declaring_supplier_id#14936235, entity_name#14936230, state#14936232, supplier_id#14936229, supplier_type#14936234, threshold#14936238]
(10) AdaptiveSparkPlan
Output [10]: [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, threshold#14936238, create_date#14936239]
Arguments: isFinalPlan=false
(11) Filter
Input [5]: [brand_context_id#14936228, supplier_id#14936229, supplier_type#14936234, declaring_supplier_id#14936235, threshold#14936238]
Condition : (isnotnull(brand_context_id#14936228) AND isnotnull(supplier_id#14936229))
(12) BroadcastExchange
Input [5]: [brand_context_id#14936228, supplier_id#14936229, supplier_type#14936234, declaring_supplier_id#14936235, threshold#14936238]
Arguments: HashedRelationBroadcastMode(List(input[0, string, false], input[1, string, false]),false), [plan_id=4296292]
(13) InMemoryTableScan
Output [4]: [brand_context_id#14938177, canonical_id#14938185, is_primary#14938186, supplier_id#14938178]
Arguments: [brand_context_id#14938177, canonical_id#14938185, is_primary#14938186, supplier_id#14938178], [isnotnull(is_primary#14938186), is_primary#14938186, isnotnull(canonical_id#14938185), isnotnull(brand_context_id#14938177), isnotnull(supplier_id#14938178)]
(14) InMemoryRelation
Arguments: [brand_context_id#14938177, supplier_id#14938178, canonical_id#14938185, is_primary#14938186, entity_name#14938179, city#14938180, state#14938181, country#14938182, threshold#14938187, create_date#14938188], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@730ae8bf,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Filter isnotnull(canonical_id#14936236)
+- TableCacheQueryStage 0
+- InMemoryTableScan [brand_context_id#14936228, supplier_id#14936229, canonical_id#14936236, is_primary#14936237, entity_name#14936230, city#14936231, state#14936232, country#14936233, threshold#14936238, create_date#14936239], [isnotnull(canonical_id#14936236)]
+- InMemoryRelation [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, canonical_id#14936236, is_primary#14936237, threshold#14936238, create_date#14936239], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14936228,supplier_id#14936229,entity_name#14936230,city#14936231,state#14936232,country#14936233,supplier_type#14936234,declaring_supplier_id#14936235,canonical_id#14936236,is_primary#14936237,threshold#14936238,create_date#14936239] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Filter isnotnull(canonical_id#14936236)
+- InMemoryTableScan [brand_context_id#14936228, supplier_id#14936229, canonical_id#14936236, is_primary#14936237, entity_name#14936230, city#14936231, state#14936232, country#14936233, threshold#14936238, create_date#14936239], [isnotnull(canonical_id#14936236)]
+- InMemoryRelation [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, canonical_id#14936236, is_primary#14936237, threshold#14936238, create_date#14936239], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14936228,supplier_id#14936229,entity_name#14936230,city#14936231,state#14936232,country#14936233,supplier_type#14936234,declaring_supplier_id#14936235,canonical_id#14936236,is_primary#14936237,threshold#14936238,create_date#14936239] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(15) InMemoryTableScan
Output [10]: [brand_context_id#14936228, supplier_id#14936229, canonical_id#14936236, is_primary#14936237, entity_name#14936230, city#14936231, state#14936232, country#14936233, threshold#14936238, create_date#14936239]
Arguments: [brand_context_id#14936228, supplier_id#14936229, canonical_id#14936236, is_primary#14936237, entity_name#14936230, city#14936231, state#14936232, country#14936233, threshold#14936238, create_date#14936239], [isnotnull(canonical_id#14936236)]
(16) InMemoryRelation
Arguments: [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, canonical_id#14936236, is_primary#14936237, threshold#14936238, create_date#14936239], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@730ae8bf,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14936228,supplier_id#14936229,entity_name#14936230,city#14936231,state#14936232,country#14936233,supplier_type#14936234,declaring_supplier_id#14936235,canonical_id#14936236,is_primary#14936237,threshold#14936238,create_date#14936239] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(17) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [codegen id : 1]
Output [12]: [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, canonical_id#14936236, is_primary#14936237, threshold#14936238, create_date#14936239]
ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,country:string,supplier_type:string,declaring_supplier_id:string,canonical_id:string,is_primary:boolean,threshold:double,create_date:date>
(18) Filter
Input [10]: [brand_context_id#14936228, supplier_id#14936229, canonical_id#14936236, is_primary#14936237, entity_name#14936230, city#14936231, state#14936232, country#14936233, threshold#14936238, create_date#14936239]
Condition : isnotnull(canonical_id#14936236)
(19) AdaptiveSparkPlan
Output [10]: [brand_context_id#14936228, supplier_id#14936229, canonical_id#14936236, is_primary#14936237, entity_name#14936230, city#14936231, state#14936232, country#14936233, threshold#14936238, create_date#14936239]
Arguments: isFinalPlan=false
(20) Filter
Input [4]: [brand_context_id#14938177, canonical_id#14938185, is_primary#14938186, supplier_id#14938178]
Condition : ((((isnotnull(is_primary#14938186) AND is_primary#14938186) AND isnotnull(canonical_id#14938185)) AND isnotnull(brand_context_id#14938177)) AND isnotnull(supplier_id#14938178))
(21) Project
Output [3]: [brand_context_id#14938177, supplier_id#14938178, canonical_id#14938185]
Input [4]: [brand_context_id#14938177, canonical_id#14938185, is_primary#14938186, supplier_id#14938178]
(22) BroadcastHashJoin
Left keys [2]: [brand_context_id#14936228, supplier_id#14936229]
Right keys [2]: [brand_context_id#14938177, supplier_id#14938178]
Join type: Inner
Join condition: None
(23) Project
Output [10]: [brand_context_id#14936228, supplier_id#14936229, declaring_supplier_id#14936235, supplier_type#14936234, canonical_id#14938185, 0 AS is_primary#14938249, threshold#14936238 AS configured_threshold#14938250, 100.0 AS match_percentage#14938251, 1.0 AS confidence_score#14938252, exact_id_match AS match_reason#14938253]
Input [8]: [brand_context_id#14936228, supplier_id#14936229, supplier_type#14936234, declaring_supplier_id#14936235, threshold#14936238, brand_context_id#14938177, supplier_id#14938178, canonical_id#14938185]
(24) InMemoryTableScan
Output [9]: [brand_context_id#14938503, supplier_id#14938504, entity_name#14938505, city#14938506, state#14938507, country#14938508, supplier_type#14938509, declaring_supplier_id#14938510, threshold#14938513]
Arguments: [brand_context_id#14938503, supplier_id#14938504, entity_name#14938505, city#14938506, state#14938507, country#14938508, supplier_type#14938509, declaring_supplier_id#14938510, threshold#14938513], [isnotnull(threshold#14938513), isnotnull(brand_context_id#14938503)]
(25) InMemoryRelation
Arguments: [brand_context_id#14938503, supplier_id#14938504, entity_name#14938505, city#14938506, state#14938507, country#14938508, supplier_type#14938509, declaring_supplier_id#14938510, threshold#14938513, create_date#14938514], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@730ae8bf,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Project [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, threshold#14936238, create_date#14936239]
+- *(1) Filter isnull(canonical_id#14936236)
+- TableCacheQueryStage 0
+- InMemoryTableScan [brand_context_id#14936228, canonical_id#14936236, city#14936231, country#14936233, create_date#14936239, declaring_supplier_id#14936235, entity_name#14936230, state#14936232, supplier_id#14936229, supplier_type#14936234, threshold#14936238], [isnull(canonical_id#14936236)]
+- InMemoryRelation [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, canonical_id#14936236, is_primary#14936237, threshold#14936238, create_date#14936239], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14936228,supplier_id#14936229,entity_name#14936230,city#14936231,state#14936232,country#14936233,supplier_type#14936234,declaring_supplier_id#14936235,canonical_id#14936236,is_primary#14936237,threshold#14936238,create_date#14936239] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Project [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, threshold#14936238, create_date#14936239]
+- Filter isnull(canonical_id#14936236)
+- InMemoryTableScan [brand_context_id#14936228, canonical_id#14936236, city#14936231, country#14936233, create_date#14936239, declaring_supplier_id#14936235, entity_name#14936230, state#14936232, supplier_id#14936229, supplier_type#14936234, threshold#14936238], [isnull(canonical_id#14936236)]
+- InMemoryRelation [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, canonical_id#14936236, is_primary#14936237, threshold#14936238, create_date#14936239], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14936228,supplier_id#14936229,entity_name#14936230,city#14936231,state#14936232,country#14936233,supplier_type#14936234,declaring_supplier_id#14936235,canonical_id#14936236,is_primary#14936237,threshold#14936238,create_date#14936239] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(26) AdaptiveSparkPlan
Output [10]: [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, threshold#14936238, create_date#14936239]
Arguments: isFinalPlan=false
(27) Filter
Input [9]: [brand_context_id#14938503, supplier_id#14938504, entity_name#14938505, city#14938506, state#14938507, country#14938508, supplier_type#14938509, declaring_supplier_id#14938510, threshold#14938513]
Condition : (isnotnull(threshold#14938513) AND isnotnull(brand_context_id#14938503))
(28) BatchEvalPython
Input [9]: [brand_context_id#14938503, supplier_id#14938504, entity_name#14938505, city#14938506, state#14938507, country#14938508, supplier_type#14938509, declaring_supplier_id#14938510, threshold#14938513]
Arguments: [normalize_location(city#14938506)#14937772, normalize_location(state#14938507)#14937786, normalize_location(country#14938508)#14937801], [pythonUDF0#14939897, pythonUDF1#14939898, pythonUDF2#14939899]
(29) Filter
Input [12]: [brand_context_id#14938503, supplier_id#14938504, entity_name#14938505, city#14938506, state#14938507, country#14938508, supplier_type#14938509, declaring_supplier_id#14938510, threshold#14938513, pythonUDF0#14939897, pythonUDF1#14939898, pythonUDF2#14939899]
Condition : ((isnotnull(pythonUDF0#14939897) AND isnotnull(pythonUDF1#14939898)) AND isnotnull(pythonUDF2#14939899))
(30) Project
Output [9]: [brand_context_id#14938503, supplier_id#14938504, entity_name#14938505, city#14938506, state#14938507, country#14938508, supplier_type#14938509, declaring_supplier_id#14938510, threshold#14938513]
Input [12]: [brand_context_id#14938503, supplier_id#14938504, entity_name#14938505, city#14938506, state#14938507, country#14938508, supplier_type#14938509, declaring_supplier_id#14938510, threshold#14938513, pythonUDF0#14939897, pythonUDF1#14939898, pythonUDF2#14939899]
(31) BatchEvalPython
Input [9]: [brand_context_id#14938503, supplier_id#14938504, entity_name#14938505, city#14938506, state#14938507, country#14938508, supplier_type#14938509, declaring_supplier_id#14938510, threshold#14938513]
Arguments: [normalize_name(entity_name#14938505)#14937759, normalize_location(city#14938506)#14937772, normalize_location(state#14938507)#14937786, normalize_location(country#14938508)#14937801], [pythonUDF0#14939900, pythonUDF1#14939901, pythonUDF2#14939902, pythonUDF3#14939903]
(32) Project
Output [7]: [brand_context_id#14938503, supplier_id#14938504, supplier_type#14938509, declaring_supplier_id#14938510, threshold#14938513, pythonUDF0#14939900 AS normalized_name#14937760, concat_ws(|, coalesce(pythonUDF1#14939901, ), coalesce(pythonUDF2#14939902, ), coalesce(pythonUDF3#14939903, )) AS location_key#14937817]
Input [13]: [brand_context_id#14938503, supplier_id#14938504, entity_name#14938505, city#14938506, state#14938507, country#14938508, supplier_type#14938509, declaring_supplier_id#14938510, threshold#14938513, pythonUDF0#14939900, pythonUDF1#14939901, pythonUDF2#14939902, pythonUDF3#14939903]
(33) Exchange
Input [7]: [brand_context_id#14938503, supplier_id#14938504, supplier_type#14938509, declaring_supplier_id#14938510, threshold#14938513, normalized_name#14937760, location_key#14937817]
Arguments: hashpartitioning(brand_context_id#14938503, supplier_id#14938504, 10), ENSURE_REQUIREMENTS, [plan_id=4296300]
(34) Sort
Input [7]: [brand_context_id#14938503, supplier_id#14938504, supplier_type#14938509, declaring_supplier_id#14938510, threshold#14938513, normalized_name#14937760, location_key#14937817]
Arguments: [brand_context_id#14938503 ASC NULLS FIRST, supplier_id#14938504 ASC NULLS FIRST], false, 0
(35) InMemoryTableScan
Output [2]: [brand_context_id#14938266, supplier_id#14938267]
Arguments: [brand_context_id#14938266, supplier_id#14938267], [isnotnull(brand_context_id#14938266), isnotnull(supplier_id#14938267)]
(36) InMemoryRelation
Arguments: [brand_context_id#14938266, supplier_id#14938267, entity_name#14938268, city#14938269, state#14938270, country#14938271, supplier_type#14938272, declaring_supplier_id#14938273, threshold#14938276, create_date#14938277], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@730ae8bf,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Project [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, threshold#14936238, create_date#14936239]
+- *(1) Filter isnull(canonical_id#14936236)
+- TableCacheQueryStage 0
+- InMemoryTableScan [brand_context_id#14936228, canonical_id#14936236, city#14936231, country#14936233, create_date#14936239, declaring_supplier_id#14936235, entity_name#14936230, state#14936232, supplier_id#14936229, supplier_type#14936234, threshold#14936238], [isnull(canonical_id#14936236)]
+- InMemoryRelation [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, canonical_id#14936236, is_primary#14936237, threshold#14936238, create_date#14936239], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14936228,supplier_id#14936229,entity_name#14936230,city#14936231,state#14936232,country#14936233,supplier_type#14936234,declaring_supplier_id#14936235,canonical_id#14936236,is_primary#14936237,threshold#14936238,create_date#14936239] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Project [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, threshold#14936238, create_date#14936239]
+- Filter isnull(canonical_id#14936236)
+- InMemoryTableScan [brand_context_id#14936228, canonical_id#14936236, city#14936231, country#14936233, create_date#14936239, declaring_supplier_id#14936235, entity_name#14936230, state#14936232, supplier_id#14936229, supplier_type#14936234, threshold#14936238], [isnull(canonical_id#14936236)]
+- InMemoryRelation [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, canonical_id#14936236, is_primary#14936237, threshold#14936238, create_date#14936239], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14936228,supplier_id#14936229,entity_name#14936230,city#14936231,state#14936232,country#14936233,supplier_type#14936234,declaring_supplier_id#14936235,canonical_id#14936236,is_primary#14936237,threshold#14936238,create_date#14936239] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(37) AdaptiveSparkPlan
Output [10]: [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, threshold#14936238, create_date#14936239]
Arguments: isFinalPlan=false
(38) Filter
Input [2]: [brand_context_id#14938266, supplier_id#14938267]
Condition : (isnotnull(brand_context_id#14938266) AND isnotnull(supplier_id#14938267))
(39) BroadcastExchange
Input [2]: [brand_context_id#14938266, supplier_id#14938267]
Arguments: HashedRelationBroadcastMode(List(input[0, string, false], input[1, string, false]),false), [plan_id=4296295]
(40) InMemoryTableScan
Output [4]: [brand_context_id#14938515, canonical_id#14938523, is_primary#14938524, supplier_id#14938516]
Arguments: [brand_context_id#14938515, canonical_id#14938523, is_primary#14938524, supplier_id#14938516], [isnotnull(is_primary#14938524), is_primary#14938524, isnotnull(canonical_id#14938523), isnotnull(brand_context_id#14938515), isnotnull(supplier_id#14938516)]
(41) InMemoryRelation
Arguments: [brand_context_id#14938515, supplier_id#14938516, canonical_id#14938523, is_primary#14938524, entity_name#14938517, city#14938518, state#14938519, country#14938520, threshold#14938525, create_date#14938526], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@730ae8bf,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Filter isnotnull(canonical_id#14936236)
+- TableCacheQueryStage 0
+- InMemoryTableScan [brand_context_id#14936228, supplier_id#14936229, canonical_id#14936236, is_primary#14936237, entity_name#14936230, city#14936231, state#14936232, country#14936233, threshold#14936238, create_date#14936239], [isnotnull(canonical_id#14936236)]
+- InMemoryRelation [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, canonical_id#14936236, is_primary#14936237, threshold#14936238, create_date#14936239], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14936228,supplier_id#14936229,entity_name#14936230,city#14936231,state#14936232,country#14936233,supplier_type#14936234,declaring_supplier_id#14936235,canonical_id#14936236,is_primary#14936237,threshold#14936238,create_date#14936239] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Filter isnotnull(canonical_id#14936236)
+- InMemoryTableScan [brand_context_id#14936228, supplier_id#14936229, canonical_id#14936236, is_primary#14936237, entity_name#14936230, city#14936231, state#14936232, country#14936233, threshold#14936238, create_date#14936239], [isnotnull(canonical_id#14936236)]
+- InMemoryRelation [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, canonical_id#14936236, is_primary#14936237, threshold#14936238, create_date#14936239], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14936228,supplier_id#14936229,entity_name#14936230,city#14936231,state#14936232,country#14936233,supplier_type#14936234,declaring_supplier_id#14936235,canonical_id#14936236,is_primary#14936237,threshold#14936238,create_date#14936239] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(42) AdaptiveSparkPlan
Output [10]: [brand_context_id#14936228, supplier_id#14936229, canonical_id#14936236, is_primary#14936237, entity_name#14936230, city#14936231, state#14936232, country#14936233, threshold#14936238, create_date#14936239]
Arguments: isFinalPlan=false
(43) Filter
Input [4]: [brand_context_id#14938515, canonical_id#14938523, is_primary#14938524, supplier_id#14938516]
Condition : ((((isnotnull(is_primary#14938524) AND is_primary#14938524) AND isnotnull(canonical_id#14938523)) AND isnotnull(brand_context_id#14938515)) AND isnotnull(supplier_id#14938516))
(44) Project
Output [2]: [brand_context_id#14938515, supplier_id#14938516]
Input [4]: [brand_context_id#14938515, canonical_id#14938523, is_primary#14938524, supplier_id#14938516]
(45) BroadcastHashJoin
Left keys [2]: [brand_context_id#14938266, supplier_id#14938267]
Right keys [2]: [brand_context_id#14938515, supplier_id#14938516]
Join type: Inner
Join condition: None
(46) Project
Output [2]: [brand_context_id#14938266, supplier_id#14938267]
Input [4]: [brand_context_id#14938266, supplier_id#14938267, brand_context_id#14938515, supplier_id#14938516]
(47) Exchange
Input [2]: [brand_context_id#14938266, supplier_id#14938267]
Arguments: hashpartitioning(brand_context_id#14938266, supplier_id#14938267, 10), ENSURE_REQUIREMENTS, [plan_id=4296301]
(48) Sort
Input [2]: [brand_context_id#14938266, supplier_id#14938267]
Arguments: [brand_context_id#14938266 ASC NULLS FIRST, supplier_id#14938267 ASC NULLS FIRST], false, 0
(49) SortMergeJoin
Left keys [2]: [brand_context_id#14938503, supplier_id#14938504]
Right keys [2]: [brand_context_id#14938266, supplier_id#14938267]
Join type: LeftAnti
Join condition: None
(50) Project
Output [7]: [brand_context_id#14938503 AS u_brand_context_id#14938293, supplier_id#14938504 AS u_supplier_id#14938294, declaring_supplier_id#14938510 AS u_declaring_supplier_id#14938295, supplier_type#14938509 AS u_supplier_type#14938296, threshold#14938513 AS u_threshold#14938297, normalized_name#14937760 AS u_normalized_name#14938298, location_key#14937817 AS u_location_key#14938302]
Input [7]: [brand_context_id#14938503, supplier_id#14938504, supplier_type#14938509, declaring_supplier_id#14938510, threshold#14938513, normalized_name#14937760, location_key#14937817]
(51) BroadcastExchange
Input [7]: [u_brand_context_id#14938293, u_supplier_id#14938294, u_declaring_supplier_id#14938295, u_supplier_type#14938296, u_threshold#14938297, u_normalized_name#14938298, u_location_key#14938302]
Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[6, string, false]),false), [plan_id=4296307]
(52) InMemoryTableScan
Output [8]: [brand_context_id#14938327, supplier_id#14938328, canonical_id#14938335, entity_name#14938329, city#14938330, state#14938331, country#14938332, create_date#14938338]
Arguments: [brand_context_id#14938327, supplier_id#14938328, canonical_id#14938335, entity_name#14938329, city#14938330, state#14938331, country#14938332, create_date#14938338], [isnotnull(brand_context_id#14938327)]
(53) InMemoryRelation
Arguments: [brand_context_id#14938327, supplier_id#14938328, canonical_id#14938335, is_primary#14938336, entity_name#14938329, city#14938330, state#14938331, country#14938332, threshold#14938337, create_date#14938338], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@730ae8bf,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Filter isnotnull(canonical_id#14936236)
+- TableCacheQueryStage 0
+- InMemoryTableScan [brand_context_id#14936228, supplier_id#14936229, canonical_id#14936236, is_primary#14936237, entity_name#14936230, city#14936231, state#14936232, country#14936233, threshold#14936238, create_date#14936239], [isnotnull(canonical_id#14936236)]
+- InMemoryRelation [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, canonical_id#14936236, is_primary#14936237, threshold#14936238, create_date#14936239], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14936228,supplier_id#14936229,entity_name#14936230,city#14936231,state#14936232,country#14936233,supplier_type#14936234,declaring_supplier_id#14936235,canonical_id#14936236,is_primary#14936237,threshold#14936238,create_date#14936239] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Filter isnotnull(canonical_id#14936236)
+- InMemoryTableScan [brand_context_id#14936228, supplier_id#14936229, canonical_id#14936236, is_primary#14936237, entity_name#14936230, city#14936231, state#14936232, country#14936233, threshold#14936238, create_date#14936239], [isnotnull(canonical_id#14936236)]
+- InMemoryRelation [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, canonical_id#14936236, is_primary#14936237, threshold#14936238, create_date#14936239], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14936228,supplier_id#14936229,entity_name#14936230,city#14936231,state#14936232,country#14936233,supplier_type#14936234,declaring_supplier_id#14936235,canonical_id#14936236,is_primary#14936237,threshold#14936238,create_date#14936239] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(54) AdaptiveSparkPlan
Output [10]: [brand_context_id#14936228, supplier_id#14936229, canonical_id#14936236, is_primary#14936237, entity_name#14936230, city#14936231, state#14936232, country#14936233, threshold#14936238, create_date#14936239]
Arguments: isFinalPlan=false
(55) Filter
Input [8]: [brand_context_id#14938327, supplier_id#14938328, canonical_id#14938335, entity_name#14938329, city#14938330, state#14938331, country#14938332, create_date#14938338]
Condition : isnotnull(brand_context_id#14938327)
(56) BatchEvalPython
Input [8]: [brand_context_id#14938327, supplier_id#14938328, canonical_id#14938335, entity_name#14938329, city#14938330, state#14938331, country#14938332, create_date#14938338]
Arguments: [normalize_name(entity_name#14938329)#14937833, normalize_location(city#14938330)#14937846, normalize_location(state#14938331)#14937860, normalize_location(country#14938332)#14937875], [pythonUDF0#14939904, pythonUDF1#14939905, pythonUDF2#14939906, pythonUDF3#14939907]
(57) Project
Output [6]: [brand_context_id#14938327 AS e_brand_context_id#14938315, supplier_id#14938328 AS e_supplier_id#14938316, canonical_id#14938335 AS e_canonical_id#14938317, pythonUDF0#14939904 AS e_normalized_name#14938318, concat_ws(|, coalesce(pythonUDF1#14939905, ), coalesce(pythonUDF2#14939906, ), coalesce(pythonUDF3#14939907, )) AS e_location_key#14938319, create_date#14938338 AS e_create_date#14938320]
Input [12]: [brand_context_id#14938327, supplier_id#14938328, canonical_id#14938335, entity_name#14938329, city#14938330, state#14938331, country#14938332, create_date#14938338, pythonUDF0#14939904, pythonUDF1#14939905, pythonUDF2#14939906, pythonUDF3#14939907]
(58) BroadcastHashJoin
Left keys [2]: [u_brand_context_id#14938293, u_location_key#14938302]
Right keys [2]: [e_brand_context_id#14938315, e_location_key#14938319]
Join type: Inner
Join condition: None
(59) Project
Output [10]: [u_brand_context_id#14938293, u_supplier_id#14938294, u_declaring_supplier_id#14938295, u_supplier_type#14938296, u_threshold#14938297, u_normalized_name#14938298, e_supplier_id#14938316, e_canonical_id#14938317, e_normalized_name#14938318, e_create_date#14938320]
Input [13]: [u_brand_context_id#14938293, u_supplier_id#14938294, u_declaring_supplier_id#14938295, u_supplier_type#14938296, u_threshold#14938297, u_normalized_name#14938298, u_location_key#14938302, e_brand_context_id#14938315, e_supplier_id#14938316, e_canonical_id#14938317, e_normalized_name#14938318, e_location_key#14938319, e_create_date#14938320]
(60) BatchEvalPython
Input [10]: [u_brand_context_id#14938293, u_supplier_id#14938294, u_declaring_supplier_id#14938295, u_supplier_type#14938296, u_threshold#14938297, u_normalized_name#14938298, e_supplier_id#14938316, e_canonical_id#14938317, e_normalized_name#14938318, e_create_date#14938320]
Arguments: [calculate_similarity(u_normalized_name#14938298, e_normalized_name#14938318)#14938373], [pythonUDF0#14939908]
(61) Filter
Input [11]: [u_brand_context_id#14938293, u_supplier_id#14938294, u_declaring_supplier_id#14938295, u_supplier_type#14938296, u_threshold#14938297, u_normalized_name#14938298, e_supplier_id#14938316, e_canonical_id#14938317, e_normalized_name#14938318, e_create_date#14938320, pythonUDF0#14939908]
Condition : (cast(pythonUDF0#14939908 as double) >= u_threshold#14938297)
(62) Project
Output [10]: [u_brand_context_id#14938293, u_supplier_id#14938294, u_declaring_supplier_id#14938295, u_supplier_type#14938296, u_threshold#14938297, u_normalized_name#14938298, e_supplier_id#14938316, e_canonical_id#14938317, e_normalized_name#14938318, e_create_date#14938320]
Input [11]: [u_brand_context_id#14938293, u_supplier_id#14938294, u_declaring_supplier_id#14938295, u_supplier_type#14938296, u_threshold#14938297, u_normalized_name#14938298, e_supplier_id#14938316, e_canonical_id#14938317, e_normalized_name#14938318, e_create_date#14938320, pythonUDF0#14939908]
(63) BatchEvalPython
Input [10]: [u_brand_context_id#14938293, u_supplier_id#14938294, u_declaring_supplier_id#14938295, u_supplier_type#14938296, u_threshold#14938297, u_normalized_name#14938298, e_supplier_id#14938316, e_canonical_id#14938317, e_normalized_name#14938318, e_create_date#14938320]
Arguments: [calculate_similarity(u_normalized_name#14938298, e_normalized_name#14938318)#14938373], [pythonUDF0#14939909]
(64) Project
Output [9]: [u_brand_context_id#14938293, u_supplier_id#14938294, u_declaring_supplier_id#14938295, u_supplier_type#14938296, u_threshold#14938297, e_supplier_id#14938316, e_canonical_id#14938317, e_create_date#14938320, pythonUDF0#14939909 AS similarity#14938374]
Input [11]: [u_brand_context_id#14938293, u_supplier_id#14938294, u_declaring_supplier_id#14938295, u_supplier_type#14938296, u_threshold#14938297, u_normalized_name#14938298, e_supplier_id#14938316, e_canonical_id#14938317, e_normalized_name#14938318, e_create_date#14938320, pythonUDF0#14939909]
(65) Sort
Input [9]: [u_brand_context_id#14938293, u_supplier_id#14938294, u_declaring_supplier_id#14938295, u_supplier_type#14938296, u_threshold#14938297, e_supplier_id#14938316, e_canonical_id#14938317, e_create_date#14938320, similarity#14938374]
Arguments: [u_brand_context_id#14938293 ASC NULLS FIRST, u_supplier_id#14938294 ASC NULLS FIRST, similarity#14938374 DESC NULLS LAST, e_create_date#14938320 ASC NULLS FIRST, e_supplier_id#14938316 ASC NULLS FIRST], false, 0
(66) WindowGroupLimit
Input [9]: [u_brand_context_id#14938293, u_supplier_id#14938294, u_declaring_supplier_id#14938295, u_supplier_type#14938296, u_threshold#14938297, e_supplier_id#14938316, e_canonical_id#14938317, e_create_date#14938320, similarity#14938374]
Arguments: [u_brand_context_id#14938293, u_supplier_id#14938294], [similarity#14938374 DESC NULLS LAST, e_create_date#14938320 ASC NULLS FIRST, e_supplier_id#14938316 ASC NULLS FIRST], row_number(), 1, Partial
(67) Exchange
Input [9]: [u_brand_context_id#14938293, u_supplier_id#14938294, u_declaring_supplier_id#14938295, u_supplier_type#14938296, u_threshold#14938297, e_supplier_id#14938316, e_canonical_id#14938317, e_create_date#14938320, similarity#14938374]
Arguments: hashpartitioning(u_brand_context_id#14938293, u_supplier_id#14938294, 10), ENSURE_REQUIREMENTS, [plan_id=4296319]
(68) Sort
Input [9]: [u_brand_context_id#14938293, u_supplier_id#14938294, u_declaring_supplier_id#14938295, u_supplier_type#14938296, u_threshold#14938297, e_supplier_id#14938316, e_canonical_id#14938317, e_create_date#14938320, similarity#14938374]
Arguments: [u_brand_context_id#14938293 ASC NULLS FIRST, u_supplier_id#14938294 ASC NULLS FIRST, similarity#14938374 DESC NULLS LAST, e_create_date#14938320 ASC NULLS FIRST, e_supplier_id#14938316 ASC NULLS FIRST], false, 0
(69) WindowGroupLimit
Input [9]: [u_brand_context_id#14938293, u_supplier_id#14938294, u_declaring_supplier_id#14938295, u_supplier_type#14938296, u_threshold#14938297, e_supplier_id#14938316, e_canonical_id#14938317, e_create_date#14938320, similarity#14938374]
Arguments: [u_brand_context_id#14938293, u_supplier_id#14938294], [similarity#14938374 DESC NULLS LAST, e_create_date#14938320 ASC NULLS FIRST, e_supplier_id#14938316 ASC NULLS FIRST], row_number(), 1, Final
(70) Window
Input [9]: [u_brand_context_id#14938293, u_supplier_id#14938294, u_declaring_supplier_id#14938295, u_supplier_type#14938296, u_threshold#14938297, e_supplier_id#14938316, e_canonical_id#14938317, e_create_date#14938320, similarity#14938374]
Arguments: [row_number() windowspecdefinition(u_brand_context_id#14938293, u_supplier_id#14938294, similarity#14938374 DESC NULLS LAST, e_create_date#14938320 ASC NULLS FIRST, e_supplier_id#14938316 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank#14938394], [u_brand_context_id#14938293, u_supplier_id#14938294], [similarity#14938374 DESC NULLS LAST, e_create_date#14938320 ASC NULLS FIRST, e_supplier_id#14938316 ASC NULLS FIRST]
(71) Filter
Input [10]: [u_brand_context_id#14938293, u_supplier_id#14938294, u_declaring_supplier_id#14938295, u_supplier_type#14938296, u_threshold#14938297, e_supplier_id#14938316, e_canonical_id#14938317, e_create_date#14938320, similarity#14938374, rank#14938394]
Condition : (rank#14938394 = 1)
(72) Project
Output [10]: [u_brand_context_id#14938293 AS brand_context_id#14938414, u_supplier_id#14938294 AS supplier_id#14938415, u_declaring_supplier_id#14938295 AS declaring_supplier_id#14938416, u_supplier_type#14938296 AS supplier_type#14938417, e_canonical_id#14938317 AS canonical_id#14938418, 0 AS is_primary#14938419, u_threshold#14938297 AS configured_threshold#14938420, cast((similarity#14938374 * 100.0) as double) AS match_percentage#14938527, cast(similarity#14938374 as double) AS confidence_score#14938528, matched_existing AS match_reason#14938423]
Input [10]: [u_brand_context_id#14938293, u_supplier_id#14938294, u_declaring_supplier_id#14938295, u_supplier_type#14938296, u_threshold#14938297, e_supplier_id#14938316, e_canonical_id#14938317, e_create_date#14938320, similarity#14938374, rank#14938394]
(73) InMemoryTableScan
Output [5]: [brand_context_id#14938539, supplier_id#14938540, supplier_type#14938545, declaring_supplier_id#14938546, threshold#14938549]
Arguments: [brand_context_id#14938539, supplier_id#14938540, supplier_type#14938545, declaring_supplier_id#14938546, threshold#14938549]
(74) InMemoryRelation
Arguments: [brand_context_id#14938539, supplier_id#14938540, entity_name#14938541, city#14938542, state#14938543, country#14938544, supplier_type#14938545, declaring_supplier_id#14938546, threshold#14938549, create_date#14938550], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@730ae8bf,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Project [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, threshold#14936238, create_date#14936239]
+- *(1) Filter isnull(canonical_id#14936236)
+- TableCacheQueryStage 0
+- InMemoryTableScan [brand_context_id#14936228, canonical_id#14936236, city#14936231, country#14936233, create_date#14936239, declaring_supplier_id#14936235, entity_name#14936230, state#14936232, supplier_id#14936229, supplier_type#14936234, threshold#14936238], [isnull(canonical_id#14936236)]
+- InMemoryRelation [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, canonical_id#14936236, is_primary#14936237, threshold#14936238, create_date#14936239], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14936228,supplier_id#14936229,entity_name#14936230,city#14936231,state#14936232,country#14936233,supplier_type#14936234,declaring_supplier_id#14936235,canonical_id#14936236,is_primary#14936237,threshold#14936238,create_date#14936239] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Project [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, threshold#14936238, create_date#14936239]
+- Filter isnull(canonical_id#14936236)
+- InMemoryTableScan [brand_context_id#14936228, canonical_id#14936236, city#14936231, country#14936233, create_date#14936239, declaring_supplier_id#14936235, entity_name#14936230, state#14936232, supplier_id#14936229, supplier_type#14936234, threshold#14936238], [isnull(canonical_id#14936236)]
+- InMemoryRelation [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, canonical_id#14936236, is_primary#14936237, threshold#14936238, create_date#14936239], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14936228,supplier_id#14936229,entity_name#14936230,city#14936231,state#14936232,country#14936233,supplier_type#14936234,declaring_supplier_id#14936235,canonical_id#14936236,is_primary#14936237,threshold#14936238,create_date#14936239] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(75) AdaptiveSparkPlan
Output [10]: [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, threshold#14936238, create_date#14936239]
Arguments: isFinalPlan=false
(76) Exchange
Input [5]: [brand_context_id#14938539, supplier_id#14938540, supplier_type#14938545, declaring_supplier_id#14938546, threshold#14938549]
Arguments: hashpartitioning(brand_context_id#14938539, supplier_id#14938540, 10), ENSURE_REQUIREMENTS, [plan_id=4296330]
(77) Sort
Input [5]: [brand_context_id#14938539, supplier_id#14938540, supplier_type#14938545, declaring_supplier_id#14938546, threshold#14938549]
Arguments: [brand_context_id#14938539 ASC NULLS FIRST, supplier_id#14938540 ASC NULLS FIRST], false, 0
(78) InMemoryTableScan
Output [2]: [brand_context_id#14938551, supplier_id#14938552]
Arguments: [brand_context_id#14938551, supplier_id#14938552], [isnotnull(brand_context_id#14938551), isnotnull(supplier_id#14938552)]
(79) InMemoryRelation
Arguments: [brand_context_id#14938551, supplier_id#14938552, entity_name#14938553, city#14938554, state#14938555, country#14938556, supplier_type#14938557, declaring_supplier_id#14938558, threshold#14938561, create_date#14938562], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@730ae8bf,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Project [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, threshold#14936238, create_date#14936239]
+- *(1) Filter isnull(canonical_id#14936236)
+- TableCacheQueryStage 0
+- InMemoryTableScan [brand_context_id#14936228, canonical_id#14936236, city#14936231, country#14936233, create_date#14936239, declaring_supplier_id#14936235, entity_name#14936230, state#14936232, supplier_id#14936229, supplier_type#14936234, threshold#14936238], [isnull(canonical_id#14936236)]
+- InMemoryRelation [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, canonical_id#14936236, is_primary#14936237, threshold#14936238, create_date#14936239], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14936228,supplier_id#14936229,entity_name#14936230,city#14936231,state#14936232,country#14936233,supplier_type#14936234,declaring_supplier_id#14936235,canonical_id#14936236,is_primary#14936237,threshold#14936238,create_date#14936239] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Project [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, threshold#14936238, create_date#14936239]
+- Filter isnull(canonical_id#14936236)
+- InMemoryTableScan [brand_context_id#14936228, canonical_id#14936236, city#14936231, country#14936233, create_date#14936239, declaring_supplier_id#14936235, entity_name#14936230, state#14936232, supplier_id#14936229, supplier_type#14936234, threshold#14936238], [isnull(canonical_id#14936236)]
+- InMemoryRelation [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, canonical_id#14936236, is_primary#14936237, threshold#14936238, create_date#14936239], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14936228,supplier_id#14936229,entity_name#14936230,city#14936231,state#14936232,country#14936233,supplier_type#14936234,declaring_supplier_id#14936235,canonical_id#14936236,is_primary#14936237,threshold#14936238,create_date#14936239] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(80) AdaptiveSparkPlan
Output [10]: [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, threshold#14936238, create_date#14936239]
Arguments: isFinalPlan=false
(81) Filter
Input [2]: [brand_context_id#14938551, supplier_id#14938552]
Condition : (isnotnull(brand_context_id#14938551) AND isnotnull(supplier_id#14938552))
(82) BroadcastExchange
Input [2]: [brand_context_id#14938551, supplier_id#14938552]
Arguments: HashedRelationBroadcastMode(List(input[0, string, false], input[1, string, false]),false), [plan_id=4296325]
(83) InMemoryTableScan
Output [4]: [brand_context_id#14938563, canonical_id#14938571, is_primary#14938572, supplier_id#14938564]
Arguments: [brand_context_id#14938563, canonical_id#14938571, is_primary#14938572, supplier_id#14938564], [isnotnull(is_primary#14938572), is_primary#14938572, isnotnull(canonical_id#14938571), isnotnull(brand_context_id#14938563), isnotnull(supplier_id#14938564)]
(84) InMemoryRelation
Arguments: [brand_context_id#14938563, supplier_id#14938564, canonical_id#14938571, is_primary#14938572, entity_name#14938565, city#14938566, state#14938567, country#14938568, threshold#14938573, create_date#14938574], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@730ae8bf,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Filter isnotnull(canonical_id#14936236)
+- TableCacheQueryStage 0
+- InMemoryTableScan [brand_context_id#14936228, supplier_id#14936229, canonical_id#14936236, is_primary#14936237, entity_name#14936230, city#14936231, state#14936232, country#14936233, threshold#14936238, create_date#14936239], [isnotnull(canonical_id#14936236)]
+- InMemoryRelation [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, canonical_id#14936236, is_primary#14936237, threshold#14936238, create_date#14936239], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14936228,supplier_id#14936229,entity_name#14936230,city#14936231,state#14936232,country#14936233,supplier_type#14936234,declaring_supplier_id#14936235,canonical_id#14936236,is_primary#14936237,threshold#14936238,create_date#14936239] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Filter isnotnull(canonical_id#14936236)
+- InMemoryTableScan [brand_context_id#14936228, supplier_id#14936229, canonical_id#14936236, is_primary#14936237, entity_name#14936230, city#14936231, state#14936232, country#14936233, threshold#14936238, create_date#14936239], [isnotnull(canonical_id#14936236)]
+- InMemoryRelation [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, canonical_id#14936236, is_primary#14936237, threshold#14936238, create_date#14936239], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14936228,supplier_id#14936229,entity_name#14936230,city#14936231,state#14936232,country#14936233,supplier_type#14936234,declaring_supplier_id#14936235,canonical_id#14936236,is_primary#14936237,threshold#14936238,create_date#14936239] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(85) AdaptiveSparkPlan
Output [10]: [brand_context_id#14936228, supplier_id#14936229, canonical_id#14936236, is_primary#14936237, entity_name#14936230, city#14936231, state#14936232, country#14936233, threshold#14936238, create_date#14936239]
Arguments: isFinalPlan=false
(86) Filter
Input [4]: [brand_context_id#14938563, canonical_id#14938571, is_primary#14938572, supplier_id#14938564]
Condition : ((((isnotnull(is_primary#14938572) AND is_primary#14938572) AND isnotnull(canonical_id#14938571)) AND isnotnull(brand_context_id#14938563)) AND isnotnull(supplier_id#14938564))
(87) Project
Output [2]: [brand_context_id#14938563, supplier_id#14938564]
Input [4]: [brand_context_id#14938563, canonical_id#14938571, is_primary#14938572, supplier_id#14938564]
(88) BroadcastHashJoin
Left keys [2]: [brand_context_id#14938551, supplier_id#14938552]
Right keys [2]: [brand_context_id#14938563, supplier_id#14938564]
Join type: Inner
Join condition: None
(89) Project
Output [2]: [brand_context_id#14938551, supplier_id#14938552]
Input [4]: [brand_context_id#14938551, supplier_id#14938552, brand_context_id#14938563, supplier_id#14938564]
(90) Exchange
Input [2]: [brand_context_id#14938551, supplier_id#14938552]
Arguments: hashpartitioning(brand_context_id#14938551, supplier_id#14938552, 10), ENSURE_REQUIREMENTS, [plan_id=4296331]
(91) Sort
Input [2]: [brand_context_id#14938551, supplier_id#14938552]
Arguments: [brand_context_id#14938551 ASC NULLS FIRST, supplier_id#14938552 ASC NULLS FIRST], false, 0
(92) SortMergeJoin
Left keys [2]: [brand_context_id#14938539, supplier_id#14938540]
Right keys [2]: [brand_context_id#14938551, supplier_id#14938552]
Join type: LeftAnti
Join condition: None
(93) InMemoryTableScan
Output [7]: [brand_context_id#14938436, supplier_id#14938437, entity_name#14938438, city#14938439, state#14938440, country#14938441, threshold#14938446]
Arguments: [brand_context_id#14938436, supplier_id#14938437, entity_name#14938438, city#14938439, state#14938440, country#14938441, threshold#14938446], [isnotnull(threshold#14938446), isnotnull(brand_context_id#14938436), isnotnull(supplier_id#14938437)]
(94) InMemoryRelation
Arguments: [brand_context_id#14938436, supplier_id#14938437, entity_name#14938438, city#14938439, state#14938440, country#14938441, supplier_type#14938442, declaring_supplier_id#14938443, threshold#14938446, create_date#14938447], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@730ae8bf,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Project [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, threshold#14936238, create_date#14936239]
+- *(1) Filter isnull(canonical_id#14936236)
+- TableCacheQueryStage 0
+- InMemoryTableScan [brand_context_id#14936228, canonical_id#14936236, city#14936231, country#14936233, create_date#14936239, declaring_supplier_id#14936235, entity_name#14936230, state#14936232, supplier_id#14936229, supplier_type#14936234, threshold#14936238], [isnull(canonical_id#14936236)]
+- InMemoryRelation [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, canonical_id#14936236, is_primary#14936237, threshold#14936238, create_date#14936239], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14936228,supplier_id#14936229,entity_name#14936230,city#14936231,state#14936232,country#14936233,supplier_type#14936234,declaring_supplier_id#14936235,canonical_id#14936236,is_primary#14936237,threshold#14936238,create_date#14936239] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Project [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, threshold#14936238, create_date#14936239]
+- Filter isnull(canonical_id#14936236)
+- InMemoryTableScan [brand_context_id#14936228, canonical_id#14936236, city#14936231, country#14936233, create_date#14936239, declaring_supplier_id#14936235, entity_name#14936230, state#14936232, supplier_id#14936229, supplier_type#14936234, threshold#14936238], [isnull(canonical_id#14936236)]
+- InMemoryRelation [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, canonical_id#14936236, is_primary#14936237, threshold#14936238, create_date#14936239], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14936228,supplier_id#14936229,entity_name#14936230,city#14936231,state#14936232,country#14936233,supplier_type#14936234,declaring_supplier_id#14936235,canonical_id#14936236,is_primary#14936237,threshold#14936238,create_date#14936239] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(95) AdaptiveSparkPlan
Output [10]: [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, threshold#14936238, create_date#14936239]
Arguments: isFinalPlan=false
(96) Filter
Input [7]: [brand_context_id#14938436, supplier_id#14938437, entity_name#14938438, city#14938439, state#14938440, country#14938441, threshold#14938446]
Condition : ((isnotnull(threshold#14938446) AND isnotnull(brand_context_id#14938436)) AND isnotnull(supplier_id#14938437))
(97) BatchEvalPython
Input [7]: [brand_context_id#14938436, supplier_id#14938437, entity_name#14938438, city#14938439, state#14938440, country#14938441, threshold#14938446]
Arguments: [normalize_location(city#14938439)#14937772, normalize_location(state#14938440)#14937786, normalize_location(country#14938441)#14937801], [pythonUDF0#14939910, pythonUDF1#14939911, pythonUDF2#14939912]
(98) Filter
Input [10]: [brand_context_id#14938436, supplier_id#14938437, entity_name#14938438, city#14938439, state#14938440, country#14938441, threshold#14938446, pythonUDF0#14939910, pythonUDF1#14939911, pythonUDF2#14939912]
Condition : ((isnotnull(pythonUDF0#14939910) AND isnotnull(pythonUDF1#14939911)) AND isnotnull(pythonUDF2#14939912))
(99) Project
Output [7]: [brand_context_id#14938436, supplier_id#14938437, entity_name#14938438, city#14938439, state#14938440, country#14938441, threshold#14938446]
Input [10]: [brand_context_id#14938436, supplier_id#14938437, entity_name#14938438, city#14938439, state#14938440, country#14938441, threshold#14938446, pythonUDF0#14939910, pythonUDF1#14939911, pythonUDF2#14939912]
(100) BatchEvalPython
Input [7]: [brand_context_id#14938436, supplier_id#14938437, entity_name#14938438, city#14938439, state#14938440, country#14938441, threshold#14938446]
Arguments: [normalize_name(entity_name#14938438)#14937759, normalize_location(city#14938439)#14937772, normalize_location(state#14938440)#14937786, normalize_location(country#14938441)#14937801], [pythonUDF0#14939913, pythonUDF1#14939914, pythonUDF2#14939915, pythonUDF3#14939916]
(101) Project
Output [5]: [brand_context_id#14938436, supplier_id#14938437, threshold#14938446, pythonUDF0#14939913 AS normalized_name#14937760, concat_ws(|, coalesce(pythonUDF1#14939914, ), coalesce(pythonUDF2#14939915, ), coalesce(pythonUDF3#14939916, )) AS location_key#14937817]
Input [11]: [brand_context_id#14938436, supplier_id#14938437, entity_name#14938438, city#14938439, state#14938440, country#14938441, threshold#14938446, pythonUDF0#14939913, pythonUDF1#14939914, pythonUDF2#14939915, pythonUDF3#14939916]
(102) Exchange
Input [5]: [brand_context_id#14938436, supplier_id#14938437, threshold#14938446, normalized_name#14937760, location_key#14937817]
Arguments: hashpartitioning(brand_context_id#14938436, supplier_id#14938437, 10), ENSURE_REQUIREMENTS, [plan_id=4296340]
(103) Sort
Input [5]: [brand_context_id#14938436, supplier_id#14938437, threshold#14938446, normalized_name#14937760, location_key#14937817]
Arguments: [brand_context_id#14938436 ASC NULLS FIRST, supplier_id#14938437 ASC NULLS FIRST], false, 0
(104) InMemoryTableScan
Output [2]: [brand_context_id#14938448, supplier_id#14938449]
Arguments: [brand_context_id#14938448, supplier_id#14938449], [isnotnull(brand_context_id#14938448), isnotnull(supplier_id#14938449)]
(105) InMemoryRelation
Arguments: [brand_context_id#14938448, supplier_id#14938449, entity_name#14938450, city#14938451, state#14938452, country#14938453, supplier_type#14938454, declaring_supplier_id#14938455, threshold#14938458, create_date#14938459], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@730ae8bf,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Project [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, threshold#14936238, create_date#14936239]
+- *(1) Filter isnull(canonical_id#14936236)
+- TableCacheQueryStage 0
+- InMemoryTableScan [brand_context_id#14936228, canonical_id#14936236, city#14936231, country#14936233, create_date#14936239, declaring_supplier_id#14936235, entity_name#14936230, state#14936232, supplier_id#14936229, supplier_type#14936234, threshold#14936238], [isnull(canonical_id#14936236)]
+- InMemoryRelation [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, canonical_id#14936236, is_primary#14936237, threshold#14936238, create_date#14936239], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14936228,supplier_id#14936229,entity_name#14936230,city#14936231,state#14936232,country#14936233,supplier_type#14936234,declaring_supplier_id#14936235,canonical_id#14936236,is_primary#14936237,threshold#14936238,create_date#14936239] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Project [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, threshold#14936238, create_date#14936239]
+- Filter isnull(canonical_id#14936236)
+- InMemoryTableScan [brand_context_id#14936228, canonical_id#14936236, city#14936231, country#14936233, create_date#14936239, declaring_supplier_id#14936235, entity_name#14936230, state#14936232, supplier_id#14936229, supplier_type#14936234, threshold#14936238], [isnull(canonical_id#14936236)]
+- InMemoryRelation [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, canonical_id#14936236, is_primary#14936237, threshold#14936238, create_date#14936239], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14936228,supplier_id#14936229,entity_name#14936230,city#14936231,state#14936232,country#14936233,supplier_type#14936234,declaring_supplier_id#14936235,canonical_id#14936236,is_primary#14936237,threshold#14936238,create_date#14936239] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(106) AdaptiveSparkPlan
Output [10]: [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, threshold#14936238, create_date#14936239]
Arguments: isFinalPlan=false
(107) Filter
Input [2]: [brand_context_id#14938448, supplier_id#14938449]
Condition : (isnotnull(brand_context_id#14938448) AND isnotnull(supplier_id#14938449))
(108) BroadcastExchange
Input [2]: [brand_context_id#14938448, supplier_id#14938449]
Arguments: HashedRelationBroadcastMode(List(input[0, string, false], input[1, string, false]),false), [plan_id=4296335]
(109) InMemoryTableScan
Output [4]: [brand_context_id#14938460, canonical_id#14938468, is_primary#14938469, supplier_id#14938461]
Arguments: [brand_context_id#14938460, canonical_id#14938468, is_primary#14938469, supplier_id#14938461], [isnotnull(is_primary#14938469), is_primary#14938469, isnotnull(canonical_id#14938468), isnotnull(brand_context_id#14938460), isnotnull(supplier_id#14938461)]
(110) InMemoryRelation
Arguments: [brand_context_id#14938460, supplier_id#14938461, canonical_id#14938468, is_primary#14938469, entity_name#14938462, city#14938463, state#14938464, country#14938465, threshold#14938470, create_date#14938471], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@730ae8bf,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Filter isnotnull(canonical_id#14936236)
+- TableCacheQueryStage 0
+- InMemoryTableScan [brand_context_id#14936228, supplier_id#14936229, canonical_id#14936236, is_primary#14936237, entity_name#14936230, city#14936231, state#14936232, country#14936233, threshold#14936238, create_date#14936239], [isnotnull(canonical_id#14936236)]
+- InMemoryRelation [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, canonical_id#14936236, is_primary#14936237, threshold#14936238, create_date#14936239], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14936228,supplier_id#14936229,entity_name#14936230,city#14936231,state#14936232,country#14936233,supplier_type#14936234,declaring_supplier_id#14936235,canonical_id#14936236,is_primary#14936237,threshold#14936238,create_date#14936239] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Filter isnotnull(canonical_id#14936236)
+- InMemoryTableScan [brand_context_id#14936228, supplier_id#14936229, canonical_id#14936236, is_primary#14936237, entity_name#14936230, city#14936231, state#14936232, country#14936233, threshold#14936238, create_date#14936239], [isnotnull(canonical_id#14936236)]
+- InMemoryRelation [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, canonical_id#14936236, is_primary#14936237, threshold#14936238, create_date#14936239], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14936228,supplier_id#14936229,entity_name#14936230,city#14936231,state#14936232,country#14936233,supplier_type#14936234,declaring_supplier_id#14936235,canonical_id#14936236,is_primary#14936237,threshold#14936238,create_date#14936239] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(111) AdaptiveSparkPlan
Output [10]: [brand_context_id#14936228, supplier_id#14936229, canonical_id#14936236, is_primary#14936237, entity_name#14936230, city#14936231, state#14936232, country#14936233, threshold#14936238, create_date#14936239]
Arguments: isFinalPlan=false
(112) Filter
Input [4]: [brand_context_id#14938460, canonical_id#14938468, is_primary#14938469, supplier_id#14938461]
Condition : ((((isnotnull(is_primary#14938469) AND is_primary#14938469) AND isnotnull(canonical_id#14938468)) AND isnotnull(brand_context_id#14938460)) AND isnotnull(supplier_id#14938461))
(113) Project
Output [2]: [brand_context_id#14938460, supplier_id#14938461]
Input [4]: [brand_context_id#14938460, canonical_id#14938468, is_primary#14938469, supplier_id#14938461]
(114) BroadcastHashJoin
Left keys [2]: [brand_context_id#14938448, supplier_id#14938449]
Right keys [2]: [brand_context_id#14938460, supplier_id#14938461]
Join type: Inner
Join condition: None
(115) Project
Output [2]: [brand_context_id#14938448, supplier_id#14938449]
Input [4]: [brand_context_id#14938448, supplier_id#14938449, brand_context_id#14938460, supplier_id#14938461]
(116) Exchange
Input [2]: [brand_context_id#14938448, supplier_id#14938449]
Arguments: hashpartitioning(brand_context_id#14938448, supplier_id#14938449, 10), ENSURE_REQUIREMENTS, [plan_id=4296341]
(117) Sort
Input [2]: [brand_context_id#14938448, supplier_id#14938449]
Arguments: [brand_context_id#14938448 ASC NULLS FIRST, supplier_id#14938449 ASC NULLS FIRST], false, 0
(118) SortMergeJoin
Left keys [2]: [brand_context_id#14938436, supplier_id#14938437]
Right keys [2]: [brand_context_id#14938448, supplier_id#14938449]
Join type: LeftAnti
Join condition: None
(119) Project
Output [5]: [brand_context_id#14938436 AS u_brand_context_id#14938293, supplier_id#14938437 AS u_supplier_id#14938294, threshold#14938446 AS u_threshold#14938297, normalized_name#14937760 AS u_normalized_name#14938298, location_key#14937817 AS u_location_key#14938302]
Input [5]: [brand_context_id#14938436, supplier_id#14938437, threshold#14938446, normalized_name#14937760, location_key#14937817]
(120) BroadcastExchange
Input [5]: [u_brand_context_id#14938293, u_supplier_id#14938294, u_threshold#14938297, u_normalized_name#14938298, u_location_key#14938302]
Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[4, string, false]),false), [plan_id=4296347]
(121) InMemoryTableScan
Output [7]: [brand_context_id#14938575, supplier_id#14938576, entity_name#14938577, city#14938578, state#14938579, country#14938580, create_date#14938586]
Arguments: [brand_context_id#14938575, supplier_id#14938576, entity_name#14938577, city#14938578, state#14938579, country#14938580, create_date#14938586], [isnotnull(brand_context_id#14938575)]
(122) InMemoryRelation
Arguments: [brand_context_id#14938575, supplier_id#14938576, canonical_id#14938583, is_primary#14938584, entity_name#14938577, city#14938578, state#14938579, country#14938580, threshold#14938585, create_date#14938586], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@730ae8bf,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Filter isnotnull(canonical_id#14936236)
+- TableCacheQueryStage 0
+- InMemoryTableScan [brand_context_id#14936228, supplier_id#14936229, canonical_id#14936236, is_primary#14936237, entity_name#14936230, city#14936231, state#14936232, country#14936233, threshold#14936238, create_date#14936239], [isnotnull(canonical_id#14936236)]
+- InMemoryRelation [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, canonical_id#14936236, is_primary#14936237, threshold#14936238, create_date#14936239], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14936228,supplier_id#14936229,entity_name#14936230,city#14936231,state#14936232,country#14936233,supplier_type#14936234,declaring_supplier_id#14936235,canonical_id#14936236,is_primary#14936237,threshold#14936238,create_date#14936239] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Filter isnotnull(canonical_id#14936236)
+- InMemoryTableScan [brand_context_id#14936228, supplier_id#14936229, canonical_id#14936236, is_primary#14936237, entity_name#14936230, city#14936231, state#14936232, country#14936233, threshold#14936238, create_date#14936239], [isnotnull(canonical_id#14936236)]
+- InMemoryRelation [brand_context_id#14936228, supplier_id#14936229, entity_name#14936230, city#14936231, state#14936232, country#14936233, supplier_type#14936234, declaring_supplier_id#14936235, canonical_id#14936236, is_primary#14936237, threshold#14936238, create_date#14936239], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14936228,supplier_id#14936229,entity_name#14936230,city#14936231,state#14936232,country#14936233,supplier_type#14936234,declaring_supplier_id#14936235,canonical_id#14936236,is_primary#14936237,threshold#14936238,create_date#14936239] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(123) AdaptiveSparkPlan
Output [10]: [brand_context_id#14936228, supplier_id#14936229, canonical_id#14936236, is_primary#14936237, entity_name#14936230, city#14936231, state#14936232, country#14936233, threshold#14936238, create_date#14936239]
Arguments: isFinalPlan=false
(124) Filter
Input [7]: [brand_context_id#14938575, supplier_id#14938576, entity_name#14938577, city#14938578, state#14938579, country#14938580, create_date#14938586]
Condition : isnotnull(brand_context_id#14938575)
(125) BatchEvalPython
Input [7]: [brand_context_id#14938575, supplier_id#14938576, entity_name#14938577, city#14938578, state#14938579, country#14938580, create_date#14938586]
Arguments: [normalize_name(entity_name#14938577)#14937833, normalize_location(city#14938578)#14937846, normalize_location(state#14938579)#14937860, normalize_location(country#14938580)#14937875], [pythonUDF0#14939917, pythonUDF1#14939918, pythonUDF2#14939919, pythonUDF3#14939920]
(126) Project
Output [5]: [brand_context_id#14938575 AS e_brand_context_id#14938315, supplier_id#14938576 AS e_supplier_id#14938316, pythonUDF0#14939917 AS e_normalized_name#14938318, concat_ws(|, coalesce(pythonUDF1#14939918, ), coalesce(pythonUDF2#14939919, ), coalesce(pythonUDF3#14939920, )) AS e_location_key#14938319, create_date#14938586 AS e_create_date#14938320]
Input [11]: [brand_context_id#14938575, supplier_id#14938576, entity_name#14938577, city#14938578, state#14938579, country#14938580, create_date#14938586, pythonUDF0#14939917, pythonUDF1#14939918, pythonUDF2#14939919, pythonUDF3#14939920]
(127) BroadcastHashJoin
Left keys [2]: [u_brand_context_id#14938293, u_location_key#14938302]
Right keys [2]: [e_brand_context_id#14938315, e_location_key#14938319]
Join type: Inner
Join condition: None
(128) Project
Output [7]: [u_brand_context_id#14938293, u_supplier_id#14938294, u_threshold#14938297, u_normalized_name#14938298, e_supplier_id#14938316, e_normalized_name#14938318, e_create_date#14938320]
Input [10]: [u_brand_context_id#14938293, u_supplier_id#14938294, u_threshold#14938297, u_normalized_name#14938298, u_location_key#14938302, e_brand_context_id#14938315, e_supplier_id#14938316, e_normalized_name#14938318, e_location_key#14938319, e_create_date#14938320]
(129) BatchEvalPython
Input [7]: [u_brand_context_id#14938293, u_supplier_id#14938294, u_threshold#14938297, u_normalized_name#14938298, e_supplier_id#14938316, e_normalized_name#14938318, e_create_date#14938320]
Arguments: [calculate_similarity(u_normalized_name#14938298, e_normalized_name#14938318)#14938373], [pythonUDF0#14939921]
(130) Filter
Input [8]: [u_brand_context_id#14938293, u_supplier_id#14938294, u_threshold#14938297, u_normalized_name#14938298, e_supplier_id#14938316, e_normalized_name#14938318, e_create_date#14938320, pythonUDF0#14939921]
Condition : (cast(pythonUDF0#14939921 as double) >= u_threshold#14938297)
(131) Project
Output [6]: [u_brand_context_id#14938293, u_supplier_id#14938294, u_normalized_name#14938298, e_supplier_id#14938316, e_normalized_name#14938318, e_create_date#14938320]
Input [8]: [u_brand_context_id#14938293, u_supplier_id#14938294, u_threshold#14938297, u_normalized_name#14938298, e_supplier_id#14938316, e_normalized_name#14938318, e_create_date#14938320, pythonUDF0#14939921]
(132) BatchEvalPython
Input [6]: [u_brand_context_id#14938293, u_supplier_id#14938294, u_normalized_name#14938298, e_supplier_id#14938316, e_normalized_name#14938318, e_create_date#14938320]
Arguments: [calculate_similarity(u_normalized_name#14938298, e_normalized_name#14938318)#14938373], [pythonUDF0#14939922]
(133) Project
Output [5]: [u_brand_context_id#14938293, u_supplier_id#14938294, e_supplier_id#14938316, e_create_date#14938320, pythonUDF0#14939922 AS similarity#14938374]
Input [7]: [u_brand_context_id#14938293, u_supplier_id#14938294, u_normalized_name#14938298, e_supplier_id#14938316, e_normalized_name#14938318, e_create_date#14938320, pythonUDF0#14939922]
(134) Sort
Input [5]: [u_brand_context_id#14938293, u_supplier_id#14938294, e_supplier_id#14938316, e_create_date#14938320, similarity#14938374]
Arguments: [u_brand_context_id#14938293 ASC NULLS FIRST, u_supplier_id#14938294 ASC NULLS FIRST, similarity#14938374 DESC NULLS LAST, e_create_date#14938320 ASC NULLS FIRST, e_supplier_id#14938316 ASC NULLS FIRST], false, 0
(135) WindowGroupLimit
Input [5]: [u_brand_context_id#14938293, u_supplier_id#14938294, e_supplier_id#14938316, e_create_date#14938320, similarity#14938374]
Arguments: [u_brand_context_id#14938293, u_supplier_id#14938294], [similarity#14938374 DESC NULLS LAST, e_create_date#14938320 ASC NULLS FIRST, e_supplier_id#14938316 ASC NULLS FIRST], row_number(), 1, Partial
(136) Exchange
Input [5]: [u_brand_context_id#14938293, u_supplier_id#14938294, e_supplier_id#14938316, e_create_date#14938320, similarity#14938374]
Arguments: hashpartitioning(u_brand_context_id#14938293, u_supplier_id#14938294, 10), ENSURE_REQUIREMENTS, [plan_id=4296359]
(137) Sort
Input [5]: [u_brand_context_id#14938293, u_supplier_id#14938294, e_supplier_id#14938316, e_create_date#14938320, similarity#14938374]
Arguments: [u_brand_context_id#14938293 ASC NULLS FIRST, u_supplier_id#14938294 ASC NULLS FIRST, similarity#14938374 DESC NULLS LAST, e_create_date#14938320 ASC NULLS FIRST, e_supplier_id#14938316 ASC NULLS FIRST], false, 0
(138) WindowGroupLimit
Input [5]: [u_brand_context_id#14938293, u_supplier_id#14938294, e_supplier_id#14938316, e_create_date#14938320, similarity#14938374]
Arguments: [u_brand_context_id#14938293, u_supplier_id#14938294], [similarity#14938374 DESC NULLS LAST, e_create_date#14938320 ASC NULLS FIRST, e_supplier_id#14938316 ASC NULLS FIRST], row_number(), 1, Final
(139) Window
Input [5]: [u_brand_context_id#14938293, u_supplier_id#14938294, e_supplier_id#14938316, e_create_date#14938320, similarity#14938374]
Arguments: [row_number() windowspecdefinition(u_brand_context_id#14938293, u_supplier_id#14938294, similarity#14938374 DESC NULLS LAST, e_create_date#14938320 ASC NULLS FIRST, e_supplier_id#14938316 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank#14938394], [u_brand_context_id#14938293, u_supplier_id#14938294], [similarity#14938374 DESC NULLS LAST, e_create_date#14938320 ASC NULLS FIRST, e_supplier_id#14938316 ASC NULLS FIRST]
(140) Filter
Input [6]: [u_brand_context_id#14938293, u_supplier_id#14938294, e_supplier_id#14938316, e_create_date#14938320, similarity#14938374, rank#14938394]
Condition : (rank#14938394 = 1)
(141) Project
Output [2]: [u_brand_context_id#14938293 AS brand_context_id#14938414, u_supplier_id#14938294 AS supplier_id#14938415]
Input [6]: [u_brand_context_id#14938293, u_supplier_id#14938294, e_supplier_id#14938316, e_create_date#14938320, similarity#14938374, rank#14938394]
(142) BroadcastExchange
Input [2]: [brand_context_id#14938414, supplier_id#14938415]
Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true]),false), [plan_id=4296366]
(143) BroadcastHashJoin
Left keys [2]: [brand_context_id#14938539, supplier_id#14938540]
Right keys [2]: [brand_context_id#14938414, supplier_id#14938415]
Join type: LeftAnti
Join condition: None
(144) Project
Output [10]: [brand_context_id#14938539, supplier_id#14938540, declaring_supplier_id#14938546, supplier_type#14938545, supplier_id#14938540 AS canonical_id#14938487, 1 AS is_primary#14938488, threshold#14938549 AS configured_threshold#14938489, 0.0 AS match_percentage#14938490, 0.0 AS confidence_score#14938491, new_canonical AS match_reason#14938492]
Input [5]: [brand_context_id#14938539, supplier_id#14938540, supplier_type#14938545, declaring_supplier_id#14938546, threshold#14938549]
(145) Union
(146) AdaptiveSparkPlan
Output [10]: [brand_context_id#14936228, supplier_id#14936229, declaring_supplier_id#14936235, supplier_type#14936234, canonical_id#14938185, is_primary#14938249, configured_threshold#14938250, match_percentage#14938251, confidence_score#14938252, match_reason#14938253]
Arguments: isFinalPlan=false
(147) TableCacheQueryStage
Output: []
Arguments: 0
(148) HashAggregate [codegen id : 1]
Input: []
Keys: []
Functions [1]: [partial_count(1)]
Aggregate Attributes [1]: [count#14942594L]
Results [1]: [count#14942595L]
(149) Exchange
Input [1]: [count#14942595L]
Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4296392]
(150) ShuffleQueryStage
Output [1]: [count#14942595L]
Arguments: 1
(151) HashAggregate [codegen id : 2]
Input [1]: [count#14942595L]
Keys: []
Functions [1]: [count(1)]
Aggregate Attributes [1]: [count(1)#14942441L]
Results [1]: [count(1)#14942441L AS count#14942442L]
(152) HashAggregate
Input: []
Keys: []
Functions [1]: [partial_count(1)]
Aggregate Attributes [1]: [count#14942594L]
Results [1]: [count#14942595L]
(153) Exchange
Input [1]: [count#14942595L]
Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4296289]
(154) HashAggregate
Input [1]: [count#14942595L]
Keys: []
Functions [1]: [count(1)]
Aggregate Attributes [1]: [count(1)#14942441L]
Results [1]: [count(1)#14942441L AS count#14942442L]
(155) AdaptiveSparkPlan
Output [1]: [count#14942442L]
Arguments: isFinalPlan=true