== Physical Plan ==
AdaptiveSparkPlan (152)
+- == Final Plan ==
DeserializeToObject (149)
+- * Project (148)
+- TableCacheQueryStage (147), Statistics(sizeInBytes=178.0 B, rowCount=1)
+- InMemoryTableScan (1)
+- InMemoryRelation (2)
+- AdaptiveSparkPlan (146)
+- Union (145)
:- Project (23)
: +- BroadcastHashJoin Inner BuildLeft (22)
: :- BroadcastExchange (12)
: : +- Filter (11)
: : +- InMemoryTableScan (3)
: : +- InMemoryRelation (4)
: : +- AdaptiveSparkPlan (10)
: : +- Project (9)
: : +- Filter (8)
: : +- InMemoryTableScan (5)
: : +- InMemoryRelation (6)
: : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: +- Project (21)
: +- Filter (20)
: +- InMemoryTableScan (13)
: +- InMemoryRelation (14)
: +- AdaptiveSparkPlan (19)
: +- Filter (18)
: +- InMemoryTableScan (15)
: +- InMemoryRelation (16)
: +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (17)
:- Project (72)
: +- Filter (71)
: +- Window (70)
: +- WindowGroupLimit (69)
: +- Sort (68)
: +- Exchange (67)
: +- WindowGroupLimit (66)
: +- Sort (65)
: +- Project (64)
: +- BatchEvalPython (63)
: +- Project (62)
: +- Filter (61)
: +- BatchEvalPython (60)
: +- Project (59)
: +- BroadcastHashJoin Inner BuildLeft (58)
: :- BroadcastExchange (51)
: : +- Project (50)
: : +- SortMergeJoin LeftAnti (49)
: : :- Sort (34)
: : : +- Exchange (33)
: : : +- Project (32)
: : : +- BatchEvalPython (31)
: : : +- Project (30)
: : : +- Filter (29)
: : : +- BatchEvalPython (28)
: : : +- Filter (27)
: : : +- InMemoryTableScan (24)
: : : +- InMemoryRelation (25)
: : : +- AdaptiveSparkPlan (26)
: : : +- Project (9)
: : : +- Filter (8)
: : : +- InMemoryTableScan (5)
: : : +- InMemoryRelation (6)
: : : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: : +- Sort (48)
: : +- Exchange (47)
: : +- Project (46)
: : +- BroadcastHashJoin Inner BuildLeft (45)
: : :- BroadcastExchange (39)
: : : +- Filter (38)
: : : +- InMemoryTableScan (35)
: : : +- InMemoryRelation (36)
: : : +- AdaptiveSparkPlan (37)
: : : +- Project (9)
: : : +- Filter (8)
: : : +- InMemoryTableScan (5)
: : : +- InMemoryRelation (6)
: : : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: : +- Project (44)
: : +- Filter (43)
: : +- InMemoryTableScan (40)
: : +- InMemoryRelation (41)
: : +- AdaptiveSparkPlan (42)
: : +- Filter (18)
: : +- InMemoryTableScan (15)
: : +- InMemoryRelation (16)
: : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (17)
: +- Project (57)
: +- BatchEvalPython (56)
: +- Filter (55)
: +- InMemoryTableScan (52)
: +- InMemoryRelation (53)
: +- AdaptiveSparkPlan (54)
: +- Filter (18)
: +- InMemoryTableScan (15)
: +- InMemoryRelation (16)
: +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (17)
+- Project (144)
+- BroadcastHashJoin LeftAnti BuildRight (143)
:- SortMergeJoin LeftAnti (92)
: :- Sort (77)
: : +- Exchange (76)
: : +- InMemoryTableScan (73)
: : +- InMemoryRelation (74)
: : +- AdaptiveSparkPlan (75)
: : +- Project (9)
: : +- Filter (8)
: : +- InMemoryTableScan (5)
: : +- InMemoryRelation (6)
: : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: +- Sort (91)
: +- Exchange (90)
: +- Project (89)
: +- BroadcastHashJoin Inner BuildLeft (88)
: :- BroadcastExchange (82)
: : +- Filter (81)
: : +- InMemoryTableScan (78)
: : +- InMemoryRelation (79)
: : +- AdaptiveSparkPlan (80)
: : +- Project (9)
: : +- Filter (8)
: : +- InMemoryTableScan (5)
: : +- InMemoryRelation (6)
: : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: +- Project (87)
: +- Filter (86)
: +- InMemoryTableScan (83)
: +- InMemoryRelation (84)
: +- AdaptiveSparkPlan (85)
: +- Filter (18)
: +- InMemoryTableScan (15)
: +- InMemoryRelation (16)
: +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (17)
+- BroadcastExchange (142)
+- Project (141)
+- Filter (140)
+- Window (139)
+- WindowGroupLimit (138)
+- Sort (137)
+- Exchange (136)
+- WindowGroupLimit (135)
+- Sort (134)
+- Project (133)
+- BatchEvalPython (132)
+- Project (131)
+- Filter (130)
+- BatchEvalPython (129)
+- Project (128)
+- BroadcastHashJoin Inner BuildLeft (127)
:- BroadcastExchange (120)
: +- Project (119)
: +- SortMergeJoin LeftAnti (118)
: :- Sort (103)
: : +- Exchange (102)
: : +- Project (101)
: : +- BatchEvalPython (100)
: : +- Project (99)
: : +- Filter (98)
: : +- BatchEvalPython (97)
: : +- Filter (96)
: : +- InMemoryTableScan (93)
: : +- InMemoryRelation (94)
: : +- AdaptiveSparkPlan (95)
: : +- Project (9)
: : +- Filter (8)
: : +- InMemoryTableScan (5)
: : +- InMemoryRelation (6)
: : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: +- Sort (117)
: +- Exchange (116)
: +- Project (115)
: +- BroadcastHashJoin Inner BuildLeft (114)
: :- BroadcastExchange (108)
: : +- Filter (107)
: : +- InMemoryTableScan (104)
: : +- InMemoryRelation (105)
: : +- AdaptiveSparkPlan (106)
: : +- Project (9)
: : +- Filter (8)
: : +- InMemoryTableScan (5)
: : +- InMemoryRelation (6)
: : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: +- Project (113)
: +- Filter (112)
: +- InMemoryTableScan (109)
: +- InMemoryRelation (110)
: +- AdaptiveSparkPlan (111)
: +- Filter (18)
: +- InMemoryTableScan (15)
: +- InMemoryRelation (16)
: +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (17)
+- Project (126)
+- BatchEvalPython (125)
+- Filter (124)
+- InMemoryTableScan (121)
+- InMemoryRelation (122)
+- AdaptiveSparkPlan (123)
+- Filter (18)
+- InMemoryTableScan (15)
+- InMemoryRelation (16)
+- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (17)
+- == Initial Plan ==
DeserializeToObject (151)
+- Project (150)
+- InMemoryTableScan (1)
+- InMemoryRelation (2)
+- AdaptiveSparkPlan (146)
+- Union (145)
:- Project (23)
: +- BroadcastHashJoin Inner BuildLeft (22)
: :- BroadcastExchange (12)
: : +- Filter (11)
: : +- InMemoryTableScan (3)
: : +- InMemoryRelation (4)
: : +- AdaptiveSparkPlan (10)
: : +- Project (9)
: : +- Filter (8)
: : +- InMemoryTableScan (5)
: : +- InMemoryRelation (6)
: : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: +- Project (21)
: +- Filter (20)
: +- InMemoryTableScan (13)
: +- InMemoryRelation (14)
: +- AdaptiveSparkPlan (19)
: +- Filter (18)
: +- InMemoryTableScan (15)
: +- InMemoryRelation (16)
: +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (17)
:- Project (72)
: +- Filter (71)
: +- Window (70)
: +- WindowGroupLimit (69)
: +- Sort (68)
: +- Exchange (67)
: +- WindowGroupLimit (66)
: +- Sort (65)
: +- Project (64)
: +- BatchEvalPython (63)
: +- Project (62)
: +- Filter (61)
: +- BatchEvalPython (60)
: +- Project (59)
: +- BroadcastHashJoin Inner BuildLeft (58)
: :- BroadcastExchange (51)
: : +- Project (50)
: : +- SortMergeJoin LeftAnti (49)
: : :- Sort (34)
: : : +- Exchange (33)
: : : +- Project (32)
: : : +- BatchEvalPython (31)
: : : +- Project (30)
: : : +- Filter (29)
: : : +- BatchEvalPython (28)
: : : +- Filter (27)
: : : +- InMemoryTableScan (24)
: : : +- InMemoryRelation (25)
: : : +- AdaptiveSparkPlan (26)
: : : +- Project (9)
: : : +- Filter (8)
: : : +- InMemoryTableScan (5)
: : : +- InMemoryRelation (6)
: : : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: : +- Sort (48)
: : +- Exchange (47)
: : +- Project (46)
: : +- BroadcastHashJoin Inner BuildLeft (45)
: : :- BroadcastExchange (39)
: : : +- Filter (38)
: : : +- InMemoryTableScan (35)
: : : +- InMemoryRelation (36)
: : : +- AdaptiveSparkPlan (37)
: : : +- Project (9)
: : : +- Filter (8)
: : : +- InMemoryTableScan (5)
: : : +- InMemoryRelation (6)
: : : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: : +- Project (44)
: : +- Filter (43)
: : +- InMemoryTableScan (40)
: : +- InMemoryRelation (41)
: : +- AdaptiveSparkPlan (42)
: : +- Filter (18)
: : +- InMemoryTableScan (15)
: : +- InMemoryRelation (16)
: : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (17)
: +- Project (57)
: +- BatchEvalPython (56)
: +- Filter (55)
: +- InMemoryTableScan (52)
: +- InMemoryRelation (53)
: +- AdaptiveSparkPlan (54)
: +- Filter (18)
: +- InMemoryTableScan (15)
: +- InMemoryRelation (16)
: +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (17)
+- Project (144)
+- BroadcastHashJoin LeftAnti BuildRight (143)
:- SortMergeJoin LeftAnti (92)
: :- Sort (77)
: : +- Exchange (76)
: : +- InMemoryTableScan (73)
: : +- InMemoryRelation (74)
: : +- AdaptiveSparkPlan (75)
: : +- Project (9)
: : +- Filter (8)
: : +- InMemoryTableScan (5)
: : +- InMemoryRelation (6)
: : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: +- Sort (91)
: +- Exchange (90)
: +- Project (89)
: +- BroadcastHashJoin Inner BuildLeft (88)
: :- BroadcastExchange (82)
: : +- Filter (81)
: : +- InMemoryTableScan (78)
: : +- InMemoryRelation (79)
: : +- AdaptiveSparkPlan (80)
: : +- Project (9)
: : +- Filter (8)
: : +- InMemoryTableScan (5)
: : +- InMemoryRelation (6)
: : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: +- Project (87)
: +- Filter (86)
: +- InMemoryTableScan (83)
: +- InMemoryRelation (84)
: +- AdaptiveSparkPlan (85)
: +- Filter (18)
: +- InMemoryTableScan (15)
: +- InMemoryRelation (16)
: +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (17)
+- BroadcastExchange (142)
+- Project (141)
+- Filter (140)
+- Window (139)
+- WindowGroupLimit (138)
+- Sort (137)
+- Exchange (136)
+- WindowGroupLimit (135)
+- Sort (134)
+- Project (133)
+- BatchEvalPython (132)
+- Project (131)
+- Filter (130)
+- BatchEvalPython (129)
+- Project (128)
+- BroadcastHashJoin Inner BuildLeft (127)
:- BroadcastExchange (120)
: +- Project (119)
: +- SortMergeJoin LeftAnti (118)
: :- Sort (103)
: : +- Exchange (102)
: : +- Project (101)
: : +- BatchEvalPython (100)
: : +- Project (99)
: : +- Filter (98)
: : +- BatchEvalPython (97)
: : +- Filter (96)
: : +- InMemoryTableScan (93)
: : +- InMemoryRelation (94)
: : +- AdaptiveSparkPlan (95)
: : +- Project (9)
: : +- Filter (8)
: : +- InMemoryTableScan (5)
: : +- InMemoryRelation (6)
: : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: +- Sort (117)
: +- Exchange (116)
: +- Project (115)
: +- BroadcastHashJoin Inner BuildLeft (114)
: :- BroadcastExchange (108)
: : +- Filter (107)
: : +- InMemoryTableScan (104)
: : +- InMemoryRelation (105)
: : +- AdaptiveSparkPlan (106)
: : +- Project (9)
: : +- Filter (8)
: : +- InMemoryTableScan (5)
: : +- InMemoryRelation (6)
: : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: +- Project (113)
: +- Filter (112)
: +- InMemoryTableScan (109)
: +- InMemoryRelation (110)
: +- AdaptiveSparkPlan (111)
: +- Filter (18)
: +- InMemoryTableScan (15)
: +- InMemoryRelation (16)
: +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (17)
+- Project (126)
+- BatchEvalPython (125)
+- Filter (124)
+- InMemoryTableScan (121)
+- InMemoryRelation (122)
+- AdaptiveSparkPlan (123)
+- Filter (18)
+- InMemoryTableScan (15)
+- InMemoryRelation (16)
+- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (17)
(1) InMemoryTableScan
Output [10]: [brand_context_id#14929523, canonical_id#14931480, confidence_score#14931547, configured_threshold#14931545, declaring_supplier_id#14929530, is_primary#14931544, match_percentage#14931546, match_reason#14931548, supplier_id#14929524, supplier_type#14929529]
Arguments: [brand_context_id#14929523, canonical_id#14931480, confidence_score#14931547, configured_threshold#14931545, declaring_supplier_id#14929530, is_primary#14931544, match_percentage#14931546, match_reason#14931548, supplier_id#14929524, supplier_type#14929529]
(2) InMemoryRelation
Arguments: [brand_context_id#14929523, supplier_id#14929524, declaring_supplier_id#14929530, supplier_type#14929529, canonical_id#14931480, is_primary#14931544, configured_threshold#14931545, match_percentage#14931546, confidence_score#14931547, match_reason#14931548], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@730ae8bf,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
Union
:- *(26) Project [brand_context_id#14929523, supplier_id#14929524, declaring_supplier_id#14929530, supplier_type#14929529, canonical_id#14931480, 0 AS is_primary#14931544, threshold#14929533 AS configured_threshold#14931545, 100.0 AS match_percentage#14931546, 1.0 AS confidence_score#14931547, exact_id_match AS match_reason#14931548]
: +- *(26) BroadcastHashJoin [brand_context_id#14929523, supplier_id#14929524], [brand_context_id#14931472, supplier_id#14931473], Inner, BuildLeft, false
: :- BroadcastQueryStage 13
: : +- BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, false], input[1, string, false]),false), [plan_id=4287632]
: : +- *(1) Filter (isnotnull(brand_context_id#14929523) AND isnotnull(supplier_id#14929524))
: : +- TableCacheQueryStage 0
: : +- InMemoryTableScan [brand_context_id#14929523, supplier_id#14929524, supplier_type#14929529, declaring_supplier_id#14929530, threshold#14929533], [isnotnull(brand_context_id#14929523), isnotnull(supplier_id#14929524)]
: : +- InMemoryRelation [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, threshold#14929533, create_date#14929534], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- AdaptiveSparkPlan isFinalPlan=false
: : +- Project [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, threshold#14929533, create_date#14929534]
: : +- Filter isnull(canonical_id#14929531)
: : +- InMemoryTableScan [brand_context_id#14929523, canonical_id#14929531, city#14929526, country#14929528, create_date#14929534, declaring_supplier_id#14929530, entity_name#14929525, state#14929527, supplier_id#14929524, supplier_type#14929529, threshold#14929533], [isnull(canonical_id#14929531)]
: : +- InMemoryRelation [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, canonical_id#14929531, is_primary#14929532, threshold#14929533, create_date#14929534], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14929523,supplier_id#14929524,entity_name#14929525,city#14929526,state#14929527,country#14929528,supplier_type#14929529,declaring_supplier_id#14929530,canonical_id#14929531,is_primary#14929532,threshold#14929533,create_date#14929534] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
: +- *(26) Project [brand_context_id#14931472, supplier_id#14931473, canonical_id#14931480]
: +- *(26) Filter ((((isnotnull(is_primary#14931481) AND is_primary#14931481) AND isnotnull(canonical_id#14931480)) AND isnotnull(brand_context_id#14931472)) AND isnotnull(supplier_id#14931473))
: +- TableCacheQueryStage 1
: +- InMemoryTableScan [brand_context_id#14931472, canonical_id#14931480, is_primary#14931481, supplier_id#14931473], [isnotnull(is_primary#14931481), is_primary#14931481, isnotnull(canonical_id#14931480), isnotnull(brand_context_id#14931472), isnotnull(supplier_id#14931473)]
: +- InMemoryRelation [brand_context_id#14931472, supplier_id#14931473, canonical_id#14931480, is_primary#14931481, entity_name#14931474, city#14931475, state#14931476, country#14931477, threshold#14931482, create_date#14931483], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- AdaptiveSparkPlan isFinalPlan=false
: +- Filter isnotnull(canonical_id#14929531)
: +- InMemoryTableScan [brand_context_id#14929523, supplier_id#14929524, canonical_id#14929531, is_primary#14929532, entity_name#14929525, city#14929526, state#14929527, country#14929528, threshold#14929533, create_date#14929534], [isnotnull(canonical_id#14929531)]
: +- InMemoryRelation [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, canonical_id#14929531, is_primary#14929532, threshold#14929533, create_date#14929534], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14929523,supplier_id#14929524,entity_name#14929525,city#14929526,state#14929527,country#14929528,supplier_type#14929529,declaring_supplier_id#14929530,canonical_id#14929531,is_primary#14929532,threshold#14929533,create_date#14929534] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
:- *(28) Project [u_brand_context_id#14931588 AS brand_context_id#14931709, u_supplier_id#14931589 AS supplier_id#14931710, u_declaring_supplier_id#14931590 AS declaring_supplier_id#14931711, u_supplier_type#14931591 AS supplier_type#14931712, e_canonical_id#14931612 AS canonical_id#14931713, 0 AS is_primary#14931714, u_threshold#14931592 AS configured_threshold#14931715, cast((similarity#14931669 * 100.0) as double) AS match_percentage#14931822, cast(similarity#14931669 as double) AS confidence_score#14931823, matched_existing AS match_reason#14931718]
: +- *(28) Filter (rank#14931689 = 1)
: +- Window [row_number() windowspecdefinition(u_brand_context_id#14931588, u_supplier_id#14931589, similarity#14931669 DESC NULLS LAST, e_create_date#14931615 ASC NULLS FIRST, e_supplier_id#14931611 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank#14931689], [u_brand_context_id#14931588, u_supplier_id#14931589], [similarity#14931669 DESC NULLS LAST, e_create_date#14931615 ASC NULLS FIRST, e_supplier_id#14931611 ASC NULLS FIRST]
: +- WindowGroupLimit [u_brand_context_id#14931588, u_supplier_id#14931589], [similarity#14931669 DESC NULLS LAST, e_create_date#14931615 ASC NULLS FIRST, e_supplier_id#14931611 ASC NULLS FIRST], row_number(), 1, Final
: +- *(27) Sort [u_brand_context_id#14931588 ASC NULLS FIRST, u_supplier_id#14931589 ASC NULLS FIRST, similarity#14931669 DESC NULLS LAST, e_create_date#14931615 ASC NULLS FIRST, e_supplier_id#14931611 ASC NULLS FIRST], false, 0
: +- ShuffleQueryStage 26
: +- Exchange hashpartitioning(u_brand_context_id#14931588, u_supplier_id#14931589, 10), ENSURE_REQUIREMENTS, [plan_id=4290711]
: +- WindowGroupLimit [u_brand_context_id#14931588, u_supplier_id#14931589], [similarity#14931669 DESC NULLS LAST, e_create_date#14931615 ASC NULLS FIRST, e_supplier_id#14931611 ASC NULLS FIRST], row_number(), 1, Partial
: +- *(23) Sort [u_brand_context_id#14931588 ASC NULLS FIRST, u_supplier_id#14931589 ASC NULLS FIRST, similarity#14931669 DESC NULLS LAST, e_create_date#14931615 ASC NULLS FIRST, e_supplier_id#14931611 ASC NULLS FIRST], false, 0
: +- *(23) Project [u_brand_context_id#14931588, u_supplier_id#14931589, u_declaring_supplier_id#14931590, u_supplier_type#14931591, u_threshold#14931592, e_supplier_id#14931611, e_canonical_id#14931612, e_create_date#14931615, pythonUDF0#14933204 AS similarity#14931669]
: +- BatchEvalPython [calculate_similarity(u_normalized_name#14931593, e_normalized_name#14931613)#14931668], [pythonUDF0#14933204]
: +- *(22) Project [u_brand_context_id#14931588, u_supplier_id#14931589, u_declaring_supplier_id#14931590, u_supplier_type#14931591, u_threshold#14931592, u_normalized_name#14931593, e_supplier_id#14931611, e_canonical_id#14931612, e_normalized_name#14931613, e_create_date#14931615]
: +- *(22) Filter (cast(pythonUDF0#14933203 as double) >= u_threshold#14931592)
: +- BatchEvalPython [calculate_similarity(u_normalized_name#14931593, e_normalized_name#14931613)#14931668], [pythonUDF0#14933203]
: +- *(21) Project [u_brand_context_id#14931588, u_supplier_id#14931589, u_declaring_supplier_id#14931590, u_supplier_type#14931591, u_threshold#14931592, u_normalized_name#14931593, e_supplier_id#14931611, e_canonical_id#14931612, e_normalized_name#14931613, e_create_date#14931615]
: +- *(21) BroadcastHashJoin [u_brand_context_id#14931588, u_location_key#14931597], [e_brand_context_id#14931610, e_location_key#14931614], Inner, BuildLeft, false
: :- BroadcastQueryStage 24
: : +- BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, true], input[6, string, false]),false), [plan_id=4290333]
: : +- *(15) Project [brand_context_id#14931798 AS u_brand_context_id#14931588, supplier_id#14931799 AS u_supplier_id#14931589, declaring_supplier_id#14931805 AS u_declaring_supplier_id#14931590, supplier_type#14931804 AS u_supplier_type#14931591, threshold#14931808 AS u_threshold#14931592, normalized_name#14931055 AS u_normalized_name#14931593, location_key#14931112 AS u_location_key#14931597]
: : +- AQEShuffleRead coalesced
: : +- ShuffleQueryStage 14
: : +- Exchange hashpartitioning(brand_context_id#14931798, supplier_id#14931799, 10), ENSURE_REQUIREMENTS, [plan_id=4287714]
: : +- *(4) Project [brand_context_id#14931798, supplier_id#14931799, supplier_type#14931804, declaring_supplier_id#14931805, threshold#14931808, pythonUDF0#14933195 AS normalized_name#14931055, concat_ws(|, coalesce(pythonUDF1#14933196, ), coalesce(pythonUDF2#14933197, ), coalesce(pythonUDF3#14933198, )) AS location_key#14931112]
: : +- BatchEvalPython [normalize_name(entity_name#14931800)#14931054, normalize_location(city#14931801)#14931067, normalize_location(state#14931802)#14931081, normalize_location(country#14931803)#14931096], [pythonUDF0#14933195, pythonUDF1#14933196, pythonUDF2#14933197, pythonUDF3#14933198]
: : +- *(3) Project [brand_context_id#14931798, supplier_id#14931799, entity_name#14931800, city#14931801, state#14931802, country#14931803, supplier_type#14931804, declaring_supplier_id#14931805, threshold#14931808]
: : +- *(3) Filter ((isnotnull(pythonUDF0#14933192) AND isnotnull(pythonUDF1#14933193)) AND isnotnull(pythonUDF2#14933194))
: : +- BatchEvalPython [normalize_location(city#14931801)#14931067, normalize_location(state#14931802)#14931081, normalize_location(country#14931803)#14931096], [pythonUDF0#14933192, pythonUDF1#14933193, pythonUDF2#14933194]
: : +- *(2) Filter (isnotnull(threshold#14931808) AND isnotnull(brand_context_id#14931798))
: : +- TableCacheQueryStage 2
: : +- InMemoryTableScan [brand_context_id#14931798, supplier_id#14931799, entity_name#14931800, city#14931801, state#14931802, country#14931803, supplier_type#14931804, declaring_supplier_id#14931805, threshold#14931808], [isnotnull(threshold#14931808), isnotnull(brand_context_id#14931798)]
: : +- InMemoryRelation [brand_context_id#14931798, supplier_id#14931799, entity_name#14931800, city#14931801, state#14931802, country#14931803, supplier_type#14931804, declaring_supplier_id#14931805, threshold#14931808, create_date#14931809], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- AdaptiveSparkPlan isFinalPlan=false
: : +- Project [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, threshold#14929533, create_date#14929534]
: : +- Filter isnull(canonical_id#14929531)
: : +- InMemoryTableScan [brand_context_id#14929523, canonical_id#14929531, city#14929526, country#14929528, create_date#14929534, declaring_supplier_id#14929530, entity_name#14929525, state#14929527, supplier_id#14929524, supplier_type#14929529, threshold#14929533], [isnull(canonical_id#14929531)]
: : +- InMemoryRelation [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, canonical_id#14929531, is_primary#14929532, threshold#14929533, create_date#14929534], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14929523,supplier_id#14929524,entity_name#14929525,city#14929526,state#14929527,country#14929528,supplier_type#14929529,declaring_supplier_id#14929530,canonical_id#14929531,is_primary#14929532,threshold#14929533,create_date#14929534] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
: +- *(21) Project [brand_context_id#14931622 AS e_brand_context_id#14931610, supplier_id#14931623 AS e_supplier_id#14931611, canonical_id#14931630 AS e_canonical_id#14931612, pythonUDF0#14933199 AS e_normalized_name#14931613, concat_ws(|, coalesce(pythonUDF1#14933200, ), coalesce(pythonUDF2#14933201, ), coalesce(pythonUDF3#14933202, )) AS e_location_key#14931614, create_date#14931633 AS e_create_date#14931615]
: +- BatchEvalPython [normalize_name(entity_name#14931624)#14931128, normalize_location(city#14931625)#14931141, normalize_location(state#14931626)#14931155, normalize_location(country#14931627)#14931170], [pythonUDF0#14933199, pythonUDF1#14933200, pythonUDF2#14933201, pythonUDF3#14933202]
: +- *(20) Filter isnotnull(brand_context_id#14931622)
: +- TableCacheQueryStage 5
: +- InMemoryTableScan [brand_context_id#14931622, supplier_id#14931623, canonical_id#14931630, entity_name#14931624, city#14931625, state#14931626, country#14931627, create_date#14931633], [isnotnull(brand_context_id#14931622)]
: +- InMemoryRelation [brand_context_id#14931622, supplier_id#14931623, canonical_id#14931630, is_primary#14931631, entity_name#14931624, city#14931625, state#14931626, country#14931627, threshold#14931632, create_date#14931633], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- AdaptiveSparkPlan isFinalPlan=false
: +- Filter isnotnull(canonical_id#14929531)
: +- InMemoryTableScan [brand_context_id#14929523, supplier_id#14929524, canonical_id#14929531, is_primary#14929532, entity_name#14929525, city#14929526, state#14929527, country#14929528, threshold#14929533, create_date#14929534], [isnotnull(canonical_id#14929531)]
: +- InMemoryRelation [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, canonical_id#14929531, is_primary#14929532, threshold#14929533, create_date#14929534], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14929523,supplier_id#14929524,entity_name#14929525,city#14929526,state#14929527,country#14929528,supplier_type#14929529,declaring_supplier_id#14929530,canonical_id#14929531,is_primary#14929532,threshold#14929533,create_date#14929534] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- *(29) Project [brand_context_id#14931834, supplier_id#14931835, declaring_supplier_id#14931841, supplier_type#14931840, supplier_id#14931835 AS canonical_id#14931782, 1 AS is_primary#14931783, threshold#14931844 AS configured_threshold#14931784, 0.0 AS match_percentage#14931785, 0.0 AS confidence_score#14931786, new_canonical AS match_reason#14931787]
+- ShuffleQueryStage 16
+- Exchange hashpartitioning(brand_context_id#14931834, supplier_id#14931835, 10), ENSURE_REQUIREMENTS, [plan_id=4287213]
+- TableCacheQueryStage 6
+- InMemoryTableScan [brand_context_id#14931834, supplier_id#14931835, supplier_type#14931840, declaring_supplier_id#14931841, threshold#14931844]
+- InMemoryRelation [brand_context_id#14931834, supplier_id#14931835, entity_name#14931836, city#14931837, state#14931838, country#14931839, supplier_type#14931840, declaring_supplier_id#14931841, threshold#14931844, create_date#14931845], StorageLevel(disk, memory, deserialized, 1 replicas)
+- AdaptiveSparkPlan isFinalPlan=false
+- Project [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, threshold#14929533, create_date#14929534]
+- Filter isnull(canonical_id#14929531)
+- InMemoryTableScan [brand_context_id#14929523, canonical_id#14929531, city#14929526, country#14929528, create_date#14929534, declaring_supplier_id#14929530, entity_name#14929525, state#14929527, supplier_id#14929524, supplier_type#14929529, threshold#14929533], [isnull(canonical_id#14929531)]
+- InMemoryRelation [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, canonical_id#14929531, is_primary#14929532, threshold#14929533, create_date#14929534], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14929523,supplier_id#14929524,entity_name#14929525,city#14929526,state#14929527,country#14929528,supplier_type#14929529,declaring_supplier_id#14929530,canonical_id#14929531,is_primary#14929532,threshold#14929533,create_date#14929534] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Union
:- Project [brand_context_id#14929523, supplier_id#14929524, declaring_supplier_id#14929530, supplier_type#14929529, canonical_id#14931480, 0 AS is_primary#14931544, threshold#14929533 AS configured_threshold#14931545, 100.0 AS match_percentage#14931546, 1.0 AS confidence_score#14931547, exact_id_match AS match_reason#14931548]
: +- BroadcastHashJoin [brand_context_id#14929523, supplier_id#14929524], [brand_context_id#14931472, supplier_id#14931473], Inner, BuildLeft, false
: :- BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, false], input[1, string, false]),false), [plan_id=4286899]
: : +- Filter (isnotnull(brand_context_id#14929523) AND isnotnull(supplier_id#14929524))
: : +- InMemoryTableScan [brand_context_id#14929523, supplier_id#14929524, supplier_type#14929529, declaring_supplier_id#14929530, threshold#14929533], [isnotnull(brand_context_id#14929523), isnotnull(supplier_id#14929524)]
: : +- InMemoryRelation [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, threshold#14929533, create_date#14929534], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- AdaptiveSparkPlan isFinalPlan=false
: : +- Project [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, threshold#14929533, create_date#14929534]
: : +- Filter isnull(canonical_id#14929531)
: : +- InMemoryTableScan [brand_context_id#14929523, canonical_id#14929531, city#14929526, country#14929528, create_date#14929534, declaring_supplier_id#14929530, entity_name#14929525, state#14929527, supplier_id#14929524, supplier_type#14929529, threshold#14929533], [isnull(canonical_id#14929531)]
: : +- InMemoryRelation [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, canonical_id#14929531, is_primary#14929532, threshold#14929533, create_date#14929534], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14929523,supplier_id#14929524,entity_name#14929525,city#14929526,state#14929527,country#14929528,supplier_type#14929529,declaring_supplier_id#14929530,canonical_id#14929531,is_primary#14929532,threshold#14929533,create_date#14929534] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
: +- Project [brand_context_id#14931472, supplier_id#14931473, canonical_id#14931480]
: +- Filter ((((isnotnull(is_primary#14931481) AND is_primary#14931481) AND isnotnull(canonical_id#14931480)) AND isnotnull(brand_context_id#14931472)) AND isnotnull(supplier_id#14931473))
: +- InMemoryTableScan [brand_context_id#14931472, canonical_id#14931480, is_primary#14931481, supplier_id#14931473], [isnotnull(is_primary#14931481), is_primary#14931481, isnotnull(canonical_id#14931480), isnotnull(brand_context_id#14931472), isnotnull(supplier_id#14931473)]
: +- InMemoryRelation [brand_context_id#14931472, supplier_id#14931473, canonical_id#14931480, is_primary#14931481, entity_name#14931474, city#14931475, state#14931476, country#14931477, threshold#14931482, create_date#14931483], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- AdaptiveSparkPlan isFinalPlan=false
: +- Filter isnotnull(canonical_id#14929531)
: +- InMemoryTableScan [brand_context_id#14929523, supplier_id#14929524, canonical_id#14929531, is_primary#14929532, entity_name#14929525, city#14929526, state#14929527, country#14929528, threshold#14929533, create_date#14929534], [isnotnull(canonical_id#14929531)]
: +- InMemoryRelation [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, canonical_id#14929531, is_primary#14929532, threshold#14929533, create_date#14929534], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14929523,supplier_id#14929524,entity_name#14929525,city#14929526,state#14929527,country#14929528,supplier_type#14929529,declaring_supplier_id#14929530,canonical_id#14929531,is_primary#14929532,threshold#14929533,create_date#14929534] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
:- Project [u_brand_context_id#14931588 AS brand_context_id#14931709, u_supplier_id#14931589 AS supplier_id#14931710, u_declaring_supplier_id#14931590 AS declaring_supplier_id#14931711, u_supplier_type#14931591 AS supplier_type#14931712, e_canonical_id#14931612 AS canonical_id#14931713, 0 AS is_primary#14931714, u_threshold#14931592 AS configured_threshold#14931715, cast((similarity#14931669 * 100.0) as double) AS match_percentage#14931822, cast(similarity#14931669 as double) AS confidence_score#14931823, matched_existing AS match_reason#14931718]
: +- Filter (rank#14931689 = 1)
: +- Window [row_number() windowspecdefinition(u_brand_context_id#14931588, u_supplier_id#14931589, similarity#14931669 DESC NULLS LAST, e_create_date#14931615 ASC NULLS FIRST, e_supplier_id#14931611 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank#14931689], [u_brand_context_id#14931588, u_supplier_id#14931589], [similarity#14931669 DESC NULLS LAST, e_create_date#14931615 ASC NULLS FIRST, e_supplier_id#14931611 ASC NULLS FIRST]
: +- WindowGroupLimit [u_brand_context_id#14931588, u_supplier_id#14931589], [similarity#14931669 DESC NULLS LAST, e_create_date#14931615 ASC NULLS FIRST, e_supplier_id#14931611 ASC NULLS FIRST], row_number(), 1, Final
: +- Sort [u_brand_context_id#14931588 ASC NULLS FIRST, u_supplier_id#14931589 ASC NULLS FIRST, similarity#14931669 DESC NULLS LAST, e_create_date#14931615 ASC NULLS FIRST, e_supplier_id#14931611 ASC NULLS FIRST], false, 0
: +- Exchange hashpartitioning(u_brand_context_id#14931588, u_supplier_id#14931589, 10), ENSURE_REQUIREMENTS, [plan_id=4286926]
: +- WindowGroupLimit [u_brand_context_id#14931588, u_supplier_id#14931589], [similarity#14931669 DESC NULLS LAST, e_create_date#14931615 ASC NULLS FIRST, e_supplier_id#14931611 ASC NULLS FIRST], row_number(), 1, Partial
: +- Sort [u_brand_context_id#14931588 ASC NULLS FIRST, u_supplier_id#14931589 ASC NULLS FIRST, similarity#14931669 DESC NULLS LAST, e_create_date#14931615 ASC NULLS FIRST, e_supplier_id#14931611 ASC NULLS FIRST], false, 0
: +- Project [u_brand_context_id#14931588, u_supplier_id#14931589, u_declaring_supplier_id#14931590, u_supplier_type#14931591, u_threshold#14931592, e_supplier_id#14931611, e_canonical_id#14931612, e_create_date#14931615, pythonUDF0#14933204 AS similarity#14931669]
: +- BatchEvalPython [calculate_similarity(u_normalized_name#14931593, e_normalized_name#14931613)#14931668], [pythonUDF0#14933204]
: +- Project [u_brand_context_id#14931588, u_supplier_id#14931589, u_declaring_supplier_id#14931590, u_supplier_type#14931591, u_threshold#14931592, u_normalized_name#14931593, e_supplier_id#14931611, e_canonical_id#14931612, e_normalized_name#14931613, e_create_date#14931615]
: +- Filter (cast(pythonUDF0#14933203 as double) >= u_threshold#14931592)
: +- BatchEvalPython [calculate_similarity(u_normalized_name#14931593, e_normalized_name#14931613)#14931668], [pythonUDF0#14933203]
: +- Project [u_brand_context_id#14931588, u_supplier_id#14931589, u_declaring_supplier_id#14931590, u_supplier_type#14931591, u_threshold#14931592, u_normalized_name#14931593, e_supplier_id#14931611, e_canonical_id#14931612, e_normalized_name#14931613, e_create_date#14931615]
: +- BroadcastHashJoin [u_brand_context_id#14931588, u_location_key#14931597], [e_brand_context_id#14931610, e_location_key#14931614], Inner, BuildLeft, false
: :- BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, true], input[6, string, false]),false), [plan_id=4286914]
: : +- Project [brand_context_id#14931798 AS u_brand_context_id#14931588, supplier_id#14931799 AS u_supplier_id#14931589, declaring_supplier_id#14931805 AS u_declaring_supplier_id#14931590, supplier_type#14931804 AS u_supplier_type#14931591, threshold#14931808 AS u_threshold#14931592, normalized_name#14931055 AS u_normalized_name#14931593, location_key#14931112 AS u_location_key#14931597]
: : +- SortMergeJoin [brand_context_id#14931798, supplier_id#14931799], [brand_context_id#14931561, supplier_id#14931562], LeftAnti
: : :- Sort [brand_context_id#14931798 ASC NULLS FIRST, supplier_id#14931799 ASC NULLS FIRST], false, 0
: : : +- Exchange hashpartitioning(brand_context_id#14931798, supplier_id#14931799, 10), ENSURE_REQUIREMENTS, [plan_id=4286907]
: : : +- Project [brand_context_id#14931798, supplier_id#14931799, supplier_type#14931804, declaring_supplier_id#14931805, threshold#14931808, pythonUDF0#14933195 AS normalized_name#14931055, concat_ws(|, coalesce(pythonUDF1#14933196, ), coalesce(pythonUDF2#14933197, ), coalesce(pythonUDF3#14933198, )) AS location_key#14931112]
: : : +- BatchEvalPython [normalize_name(entity_name#14931800)#14931054, normalize_location(city#14931801)#14931067, normalize_location(state#14931802)#14931081, normalize_location(country#14931803)#14931096], [pythonUDF0#14933195, pythonUDF1#14933196, pythonUDF2#14933197, pythonUDF3#14933198]
: : : +- Project [brand_context_id#14931798, supplier_id#14931799, entity_name#14931800, city#14931801, state#14931802, country#14931803, supplier_type#14931804, declaring_supplier_id#14931805, threshold#14931808]
: : : +- Filter ((isnotnull(pythonUDF0#14933192) AND isnotnull(pythonUDF1#14933193)) AND isnotnull(pythonUDF2#14933194))
: : : +- BatchEvalPython [normalize_location(city#14931801)#14931067, normalize_location(state#14931802)#14931081, normalize_location(country#14931803)#14931096], [pythonUDF0#14933192, pythonUDF1#14933193, pythonUDF2#14933194]
: : : +- Filter (isnotnull(threshold#14931808) AND isnotnull(brand_context_id#14931798))
: : : +- InMemoryTableScan [brand_context_id#14931798, supplier_id#14931799, entity_name#14931800, city#14931801, state#14931802, country#14931803, supplier_type#14931804, declaring_supplier_id#14931805, threshold#14931808], [isnotnull(threshold#14931808), isnotnull(brand_context_id#14931798)]
: : : +- InMemoryRelation [brand_context_id#14931798, supplier_id#14931799, entity_name#14931800, city#14931801, state#14931802, country#14931803, supplier_type#14931804, declaring_supplier_id#14931805, threshold#14931808, create_date#14931809], StorageLevel(disk, memory, deserialized, 1 replicas)
: : : +- AdaptiveSparkPlan isFinalPlan=false
: : : +- Project [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, threshold#14929533, create_date#14929534]
: : : +- Filter isnull(canonical_id#14929531)
: : : +- InMemoryTableScan [brand_context_id#14929523, canonical_id#14929531, city#14929526, country#14929528, create_date#14929534, declaring_supplier_id#14929530, entity_name#14929525, state#14929527, supplier_id#14929524, supplier_type#14929529, threshold#14929533], [isnull(canonical_id#14929531)]
: : : +- InMemoryRelation [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, canonical_id#14929531, is_primary#14929532, threshold#14929533, create_date#14929534], StorageLevel(disk, memory, deserialized, 1 replicas)
: : : +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14929523,supplier_id#14929524,entity_name#14929525,city#14929526,state#14929527,country#14929528,supplier_type#14929529,declaring_supplier_id#14929530,canonical_id#14929531,is_primary#14929532,threshold#14929533,create_date#14929534] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
: : +- Sort [brand_context_id#14931561 ASC NULLS FIRST, supplier_id#14931562 ASC NULLS FIRST], false, 0
: : +- Exchange hashpartitioning(brand_context_id#14931561, supplier_id#14931562, 10), ENSURE_REQUIREMENTS, [plan_id=4286908]
: : +- Project [brand_context_id#14931561, supplier_id#14931562]
: : +- BroadcastHashJoin [brand_context_id#14931561, supplier_id#14931562], [brand_context_id#14931810, supplier_id#14931811], Inner, BuildLeft, false
: : :- BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, false], input[1, string, false]),false), [plan_id=4286902]
: : : +- Filter (isnotnull(brand_context_id#14931561) AND isnotnull(supplier_id#14931562))
: : : +- InMemoryTableScan [brand_context_id#14931561, supplier_id#14931562], [isnotnull(brand_context_id#14931561), isnotnull(supplier_id#14931562)]
: : : +- InMemoryRelation [brand_context_id#14931561, supplier_id#14931562, entity_name#14931563, city#14931564, state#14931565, country#14931566, supplier_type#14931567, declaring_supplier_id#14931568, threshold#14931571, create_date#14931572], StorageLevel(disk, memory, deserialized, 1 replicas)
: : : +- AdaptiveSparkPlan isFinalPlan=false
: : : +- Project [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, threshold#14929533, create_date#14929534]
: : : +- Filter isnull(canonical_id#14929531)
: : : +- InMemoryTableScan [brand_context_id#14929523, canonical_id#14929531, city#14929526, country#14929528, create_date#14929534, declaring_supplier_id#14929530, entity_name#14929525, state#14929527, supplier_id#14929524, supplier_type#14929529, threshold#14929533], [isnull(canonical_id#14929531)]
: : : +- InMemoryRelation [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, canonical_id#14929531, is_primary#14929532, threshold#14929533, create_date#14929534], StorageLevel(disk, memory, deserialized, 1 replicas)
: : : +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14929523,supplier_id#14929524,entity_name#14929525,city#14929526,state#14929527,country#14929528,supplier_type#14929529,declaring_supplier_id#14929530,canonical_id#14929531,is_primary#14929532,threshold#14929533,create_date#14929534] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
: : +- Project [brand_context_id#14931810, supplier_id#14931811]
: : +- Filter ((((isnotnull(is_primary#14931819) AND is_primary#14931819) AND isnotnull(canonical_id#14931818)) AND isnotnull(brand_context_id#14931810)) AND isnotnull(supplier_id#14931811))
: : +- InMemoryTableScan [brand_context_id#14931810, canonical_id#14931818, is_primary#14931819, supplier_id#14931811], [isnotnull(is_primary#14931819), is_primary#14931819, isnotnull(canonical_id#14931818), isnotnull(brand_context_id#14931810), isnotnull(supplier_id#14931811)]
: : +- InMemoryRelation [brand_context_id#14931810, supplier_id#14931811, canonical_id#14931818, is_primary#14931819, entity_name#14931812, city#14931813, state#14931814, country#14931815, threshold#14931820, create_date#14931821], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- AdaptiveSparkPlan isFinalPlan=false
: : +- Filter isnotnull(canonical_id#14929531)
: : +- InMemoryTableScan [brand_context_id#14929523, supplier_id#14929524, canonical_id#14929531, is_primary#14929532, entity_name#14929525, city#14929526, state#14929527, country#14929528, threshold#14929533, create_date#14929534], [isnotnull(canonical_id#14929531)]
: : +- InMemoryRelation [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, canonical_id#14929531, is_primary#14929532, threshold#14929533, create_date#14929534], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14929523,supplier_id#14929524,entity_name#14929525,city#14929526,state#14929527,country#14929528,supplier_type#14929529,declaring_supplier_id#14929530,canonical_id#14929531,is_primary#14929532,threshold#14929533,create_date#14929534] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
: +- Project [brand_context_id#14931622 AS e_brand_context_id#14931610, supplier_id#14931623 AS e_supplier_id#14931611, canonical_id#14931630 AS e_canonical_id#14931612, pythonUDF0#14933199 AS e_normalized_name#14931613, concat_ws(|, coalesce(pythonUDF1#14933200, ), coalesce(pythonUDF2#14933201, ), coalesce(pythonUDF3#14933202, )) AS e_location_key#14931614, create_date#14931633 AS e_create_date#14931615]
: +- BatchEvalPython [normalize_name(entity_name#14931624)#14931128, normalize_location(city#14931625)#14931141, normalize_location(state#14931626)#14931155, normalize_location(country#14931627)#14931170], [pythonUDF0#14933199, pythonUDF1#14933200, pythonUDF2#14933201, pythonUDF3#14933202]
: +- Filter isnotnull(brand_context_id#14931622)
: +- InMemoryTableScan [brand_context_id#14931622, supplier_id#14931623, canonical_id#14931630, entity_name#14931624, city#14931625, state#14931626, country#14931627, create_date#14931633], [isnotnull(brand_context_id#14931622)]
: +- InMemoryRelation [brand_context_id#14931622, supplier_id#14931623, canonical_id#14931630, is_primary#14931631, entity_name#14931624, city#14931625, state#14931626, country#14931627, threshold#14931632, create_date#14931633], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- AdaptiveSparkPlan isFinalPlan=false
: +- Filter isnotnull(canonical_id#14929531)
: +- InMemoryTableScan [brand_context_id#14929523, supplier_id#14929524, canonical_id#14929531, is_primary#14929532, entity_name#14929525, city#14929526, state#14929527, country#14929528, threshold#14929533, create_date#14929534], [isnotnull(canonical_id#14929531)]
: +- InMemoryRelation [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, canonical_id#14929531, is_primary#14929532, threshold#14929533, create_date#14929534], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14929523,supplier_id#14929524,entity_name#14929525,city#14929526,state#14929527,country#14929528,supplier_type#14929529,declaring_supplier_id#14929530,canonical_id#14929531,is_primary#14929532,threshold#14929533,create_date#14929534] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- Project [brand_context_id#14931834, supplier_id#14931835, declaring_supplier_id#14931841, supplier_type#14931840, supplier_id#14931835 AS canonical_id#14931782, 1 AS is_primary#14931783, threshold#14931844 AS configured_threshold#14931784, 0.0 AS match_percentage#14931785, 0.0 AS confidence_score#14931786, new_canonical AS match_reason#14931787]
+- BroadcastHashJoin [brand_context_id#14931834, supplier_id#14931835], [brand_context_id#14931709, supplier_id#14931710], LeftAnti, BuildRight, false
:- SortMergeJoin [brand_context_id#14931834, supplier_id#14931835], [brand_context_id#14931846, supplier_id#14931847], LeftAnti
: :- Sort [brand_context_id#14931834 ASC NULLS FIRST, supplier_id#14931835 ASC NULLS FIRST], false, 0
: : +- Exchange hashpartitioning(brand_context_id#14931834, supplier_id#14931835, 10), ENSURE_REQUIREMENTS, [plan_id=4286937]
: : +- InMemoryTableScan [brand_context_id#14931834, supplier_id#14931835, supplier_type#14931840, declaring_supplier_id#14931841, threshold#14931844]
: : +- InMemoryRelation [brand_context_id#14931834, supplier_id#14931835, entity_name#14931836, city#14931837, state#14931838, country#14931839, supplier_type#14931840, declaring_supplier_id#14931841, threshold#14931844, create_date#14931845], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- AdaptiveSparkPlan isFinalPlan=false
: : +- Project [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, threshold#14929533, create_date#14929534]
: : +- Filter isnull(canonical_id#14929531)
: : +- InMemoryTableScan [brand_context_id#14929523, canonical_id#14929531, city#14929526, country#14929528, create_date#14929534, declaring_supplier_id#14929530, entity_name#14929525, state#14929527, supplier_id#14929524, supplier_type#14929529, threshold#14929533], [isnull(canonical_id#14929531)]
: : +- InMemoryRelation [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, canonical_id#14929531, is_primary#14929532, threshold#14929533, create_date#14929534], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14929523,supplier_id#14929524,entity_name#14929525,city#14929526,state#14929527,country#14929528,supplier_type#14929529,declaring_supplier_id#14929530,canonical_id#14929531,is_primary#14929532,threshold#14929533,create_date#14929534] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
: +- Sort [brand_context_id#14931846 ASC NULLS FIRST, supplier_id#14931847 ASC NULLS FIRST], false, 0
: +- Exchange hashpartitioning(brand_context_id#14931846, supplier_id#14931847, 10), ENSURE_REQUIREMENTS, [plan_id=4286938]
: +- Project [brand_context_id#14931846, supplier_id#14931847]
: +- BroadcastHashJoin [brand_context_id#14931846, supplier_id#14931847], [brand_context_id#14931858, supplier_id#14931859], Inner, BuildLeft, false
: :- BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, false], input[1, string, false]),false), [plan_id=4286932]
: : +- Filter (isnotnull(brand_context_id#14931846) AND isnotnull(supplier_id#14931847))
: : +- InMemoryTableScan [brand_context_id#14931846, supplier_id#14931847], [isnotnull(brand_context_id#14931846), isnotnull(supplier_id#14931847)]
: : +- InMemoryRelation [brand_context_id#14931846, supplier_id#14931847, entity_name#14931848, city#14931849, state#14931850, country#14931851, supplier_type#14931852, declaring_supplier_id#14931853, threshold#14931856, create_date#14931857], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- AdaptiveSparkPlan isFinalPlan=false
: : +- Project [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, threshold#14929533, create_date#14929534]
: : +- Filter isnull(canonical_id#14929531)
: : +- InMemoryTableScan [brand_context_id#14929523, canonical_id#14929531, city#14929526, country#14929528, create_date#14929534, declaring_supplier_id#14929530, entity_name#14929525, state#14929527, supplier_id#14929524, supplier_type#14929529, threshold#14929533], [isnull(canonical_id#14929531)]
: : +- InMemoryRelation [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, canonical_id#14929531, is_primary#14929532, threshold#14929533, create_date#14929534], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14929523,supplier_id#14929524,entity_name#14929525,city#14929526,state#14929527,country#14929528,supplier_type#14929529,declaring_supplier_id#14929530,canonical_id#14929531,is_primary#14929532,threshold#14929533,create_date#14929534] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
: +- Project [brand_context_id#14931858, supplier_id#14931859]
: +- Filter ((((isnotnull(is_primary#14931867) AND is_primary#14931867) AND isnotnull(canonical_id#14931866)) AND isnotnull(brand_context_id#14931858)) AND isnotnull(supplier_id#14931859))
: +- InMemoryTableScan [brand_context_id#14931858, canonical_id#14931866, is_primary#14931867, supplier_id#14931859], [isnotnull(is_primary#14931867), is_primary#14931867, isnotnull(canonical_id#14931866), isnotnull(brand_context_id#14931858), isnotnull(supplier_id#14931859)]
: +- InMemoryRelation [brand_context_id#14931858, supplier_id#14931859, canonical_id#14931866, is_primary#14931867, entity_name#14931860, city#14931861, state#14931862, country#14931863, threshold#14931868, create_date#14931869], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- AdaptiveSparkPlan isFinalPlan=false
: +- Filter isnotnull(canonical_id#14929531)
: +- InMemoryTableScan [brand_context_id#14929523, supplier_id#14929524, canonical_id#14929531, is_primary#14929532, entity_name#14929525, city#14929526, state#14929527, country#14929528, threshold#14929533, create_date#14929534], [isnotnull(canonical_id#14929531)]
: +- InMemoryRelation [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, canonical_id#14929531, is_primary#14929532, threshold#14929533, create_date#14929534], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14929523,supplier_id#14929524,entity_name#14929525,city#14929526,state#14929527,country#14929528,supplier_type#14929529,declaring_supplier_id#14929530,canonical_id#14929531,is_primary#14929532,threshold#14929533,create_date#14929534] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true]),false), [plan_id=4286973]
+- Project [u_brand_context_id#14931588 AS brand_context_id#14931709, u_supplier_id#14931589 AS supplier_id#14931710]
+- Filter (rank#14931689 = 1)
+- Window [row_number() windowspecdefinition(u_brand_context_id#14931588, u_supplier_id#14931589, similarity#14931669 DESC NULLS LAST, e_create_date#14931615 ASC NULLS FIRST, e_supplier_id#14931611 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank#14931689], [u_brand_context_id#14931588, u_supplier_id#14931589], [similarity#14931669 DESC NULLS LAST, e_create_date#14931615 ASC NULLS FIRST, e_supplier_id#14931611 ASC NULLS FIRST]
+- WindowGroupLimit [u_brand_context_id#14931588, u_supplier_id#14931589], [similarity#14931669 DESC NULLS LAST, e_create_date#14931615 ASC NULLS FIRST, e_supplier_id#14931611 ASC NULLS FIRST], row_number(), 1, Final
+- Sort [u_brand_context_id#14931588 ASC NULLS FIRST, u_supplier_id#14931589 ASC NULLS FIRST, similarity#14931669 DESC NULLS LAST, e_create_date#14931615 ASC NULLS FIRST, e_supplier_id#14931611 ASC NULLS FIRST], false, 0
+- Exchange hashpartitioning(u_brand_context_id#14931588, u_supplier_id#14931589, 10), ENSURE_REQUIREMENTS, [plan_id=4286966]
+- WindowGroupLimit [u_brand_context_id#14931588, u_supplier_id#14931589], [similarity#14931669 DESC NULLS LAST, e_create_date#14931615 ASC NULLS FIRST, e_supplier_id#14931611 ASC NULLS FIRST], row_number(), 1, Partial
+- Sort [u_brand_context_id#14931588 ASC NULLS FIRST, u_supplier_id#14931589 ASC NULLS FIRST, similarity#14931669 DESC NULLS LAST, e_create_date#14931615 ASC NULLS FIRST, e_supplier_id#14931611 ASC NULLS FIRST], false, 0
+- Project [u_brand_context_id#14931588, u_supplier_id#14931589, e_supplier_id#14931611, e_create_date#14931615, pythonUDF0#14933217 AS similarity#14931669]
+- BatchEvalPython [calculate_similarity(u_normalized_name#14931593, e_normalized_name#14931613)#14931668], [pythonUDF0#14933217]
+- Project [u_brand_context_id#14931588, u_supplier_id#14931589, u_normalized_name#14931593, e_supplier_id#14931611, e_normalized_name#14931613, e_create_date#14931615]
+- Filter (cast(pythonUDF0#14933216 as double) >= u_threshold#14931592)
+- BatchEvalPython [calculate_similarity(u_normalized_name#14931593, e_normalized_name#14931613)#14931668], [pythonUDF0#14933216]
+- Project [u_brand_context_id#14931588, u_supplier_id#14931589, u_threshold#14931592, u_normalized_name#14931593, e_supplier_id#14931611, e_normalized_name#14931613, e_create_date#14931615]
+- BroadcastHashJoin [u_brand_context_id#14931588, u_location_key#14931597], [e_brand_context_id#14931610, e_location_key#14931614], Inner, BuildLeft, false
:- BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, true], input[4, string, false]),false), [plan_id=4286954]
: +- Project [brand_context_id#14931731 AS u_brand_context_id#14931588, supplier_id#14931732 AS u_supplier_id#14931589, threshold#14931741 AS u_threshold#14931592, normalized_name#14931055 AS u_normalized_name#14931593, location_key#14931112 AS u_location_key#14931597]
: +- SortMergeJoin [brand_context_id#14931731, supplier_id#14931732], [brand_context_id#14931743, supplier_id#14931744], LeftAnti
: :- Sort [brand_context_id#14931731 ASC NULLS FIRST, supplier_id#14931732 ASC NULLS FIRST], false, 0
: : +- Exchange hashpartitioning(brand_context_id#14931731, supplier_id#14931732, 10), ENSURE_REQUIREMENTS, [plan_id=4286947]
: : +- Project [brand_context_id#14931731, supplier_id#14931732, threshold#14931741, pythonUDF0#14933208 AS normalized_name#14931055, concat_ws(|, coalesce(pythonUDF1#14933209, ), coalesce(pythonUDF2#14933210, ), coalesce(pythonUDF3#14933211, )) AS location_key#14931112]
: : +- BatchEvalPython [normalize_name(entity_name#14931733)#14931054, normalize_location(city#14931734)#14931067, normalize_location(state#14931735)#14931081, normalize_location(country#14931736)#14931096], [pythonUDF0#14933208, pythonUDF1#14933209, pythonUDF2#14933210, pythonUDF3#14933211]
: : +- Project [brand_context_id#14931731, supplier_id#14931732, entity_name#14931733, city#14931734, state#14931735, country#14931736, threshold#14931741]
: : +- Filter ((isnotnull(pythonUDF0#14933205) AND isnotnull(pythonUDF1#14933206)) AND isnotnull(pythonUDF2#14933207))
: : +- BatchEvalPython [normalize_location(city#14931734)#14931067, normalize_location(state#14931735)#14931081, normalize_location(country#14931736)#14931096], [pythonUDF0#14933205, pythonUDF1#14933206, pythonUDF2#14933207]
: : +- Filter ((isnotnull(threshold#14931741) AND isnotnull(brand_context_id#14931731)) AND isnotnull(supplier_id#14931732))
: : +- InMemoryTableScan [brand_context_id#14931731, supplier_id#14931732, entity_name#14931733, city#14931734, state#14931735, country#14931736, threshold#14931741], [isnotnull(threshold#14931741), isnotnull(brand_context_id#14931731), isnotnull(supplier_id#14931732)]
: : +- InMemoryRelation [brand_context_id#14931731, supplier_id#14931732, entity_name#14931733, city#14931734, state#14931735, country#14931736, supplier_type#14931737, declaring_supplier_id#14931738, threshold#14931741, create_date#14931742], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- AdaptiveSparkPlan isFinalPlan=false
: : +- Project [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, threshold#14929533, create_date#14929534]
: : +- Filter isnull(canonical_id#14929531)
: : +- InMemoryTableScan [brand_context_id#14929523, canonical_id#14929531, city#14929526, country#14929528, create_date#14929534, declaring_supplier_id#14929530, entity_name#14929525, state#14929527, supplier_id#14929524, supplier_type#14929529, threshold#14929533], [isnull(canonical_id#14929531)]
: : +- InMemoryRelation [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, canonical_id#14929531, is_primary#14929532, threshold#14929533, create_date#14929534], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14929523,supplier_id#14929524,entity_name#14929525,city#14929526,state#14929527,country#14929528,supplier_type#14929529,declaring_supplier_id#14929530,canonical_id#14929531,is_primary#14929532,threshold#14929533,create_date#14929534] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
: +- Sort [brand_context_id#14931743 ASC NULLS FIRST, supplier_id#14931744 ASC NULLS FIRST], false, 0
: +- Exchange hashpartitioning(brand_context_id#14931743, supplier_id#14931744, 10), ENSURE_REQUIREMENTS, [plan_id=4286948]
: +- Project [brand_context_id#14931743, supplier_id#14931744]
: +- BroadcastHashJoin [brand_context_id#14931743, supplier_id#14931744], [brand_context_id#14931755, supplier_id#14931756], Inner, BuildLeft, false
: :- BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, false], input[1, string, false]),false), [plan_id=4286942]
: : +- Filter (isnotnull(brand_context_id#14931743) AND isnotnull(supplier_id#14931744))
: : +- InMemoryTableScan [brand_context_id#14931743, supplier_id#14931744], [isnotnull(brand_context_id#14931743), isnotnull(supplier_id#14931744)]
: : +- InMemoryRelation [brand_context_id#14931743, supplier_id#14931744, entity_name#14931745, city#14931746, state#14931747, country#14931748, supplier_type#14931749, declaring_supplier_id#14931750, threshold#14931753, create_date#14931754], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- AdaptiveSparkPlan isFinalPlan=false
: : +- Project [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, threshold#14929533, create_date#14929534]
: : +- Filter isnull(canonical_id#14929531)
: : +- InMemoryTableScan [brand_context_id#14929523, canonical_id#14929531, city#14929526, country#14929528, create_date#14929534, declaring_supplier_id#14929530, entity_name#14929525, state#14929527, supplier_id#14929524, supplier_type#14929529, threshold#14929533], [isnull(canonical_id#14929531)]
: : +- InMemoryRelation [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, canonical_id#14929531, is_primary#14929532, threshold#14929533, create_date#14929534], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14929523,supplier_id#14929524,entity_name#14929525,city#14929526,state#14929527,country#14929528,supplier_type#14929529,declaring_supplier_id#14929530,canonical_id#14929531,is_primary#14929532,threshold#14929533,create_date#14929534] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
: +- Project [brand_context_id#14931755, supplier_id#14931756]
: +- Filter ((((isnotnull(is_primary#14931764) AND is_primary#14931764) AND isnotnull(canonical_id#14931763)) AND isnotnull(brand_context_id#14931755)) AND isnotnull(supplier_id#14931756))
: +- InMemoryTableScan [brand_context_id#14931755, canonical_id#14931763, is_primary#14931764, supplier_id#14931756], [isnotnull(is_primary#14931764), is_primary#14931764, isnotnull(canonical_id#14931763), isnotnull(brand_context_id#14931755), isnotnull(supplier_id#14931756)]
: +- InMemoryRelation [brand_context_id#14931755, supplier_id#14931756, canonical_id#14931763, is_primary#14931764, entity_name#14931757, city#14931758, state#14931759, country#14931760, threshold#14931765, create_date#14931766], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- AdaptiveSparkPlan isFinalPlan=false
: +- Filter isnotnull(canonical_id#14929531)
: +- InMemoryTableScan [brand_context_id#14929523, supplier_id#14929524, canonical_id#14929531, is_primary#14929532, entity_name#14929525, city#14929526, state#14929527, country#14929528, threshold#14929533, create_date#14929534], [isnotnull(canonical_id#14929531)]
: +- InMemoryRelation [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, canonical_id#14929531, is_primary#14929532, threshold#14929533, create_date#14929534], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14929523,supplier_id#14929524,entity_name#14929525,city#14929526,state#14929527,country#14929528,supplier_type#14929529,declaring_supplier_id#14929530,canonical_id#14929531,is_primary#14929532,threshold#14929533,create_date#14929534] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- Project [brand_context_id#14931870 AS e_brand_context_id#14931610, supplier_id#14931871 AS e_supplier_id#14931611, pythonUDF0#14933212 AS e_normalized_name#14931613, concat_ws(|, coalesce(pythonUDF1#14933213, ), coalesce(pythonUDF2#14933214, ), coalesce(pythonUDF3#14933215, )) AS e_location_key#14931614, create_date#14931881 AS e_create_date#14931615]
+- BatchEvalPython [normalize_name(entity_name#14931872)#14931128, normalize_location(city#14931873)#14931141, normalize_location(state#14931874)#14931155, normalize_location(country#14931875)#14931170], [pythonUDF0#14933212, pythonUDF1#14933213, pythonUDF2#14933214, pythonUDF3#14933215]
+- Filter isnotnull(brand_context_id#14931870)
+- InMemoryTableScan [brand_context_id#14931870, supplier_id#14931871, entity_name#14931872, city#14931873, state#14931874, country#14931875, create_date#14931881], [isnotnull(brand_context_id#14931870)]
+- InMemoryRelation [brand_context_id#14931870, supplier_id#14931871, canonical_id#14931878, is_primary#14931879, entity_name#14931872, city#14931873, state#14931874, country#14931875, threshold#14931880, create_date#14931881], StorageLevel(disk, memory, deserialized, 1 replicas)
+- AdaptiveSparkPlan isFinalPlan=false
+- Filter isnotnull(canonical_id#14929531)
+- InMemoryTableScan [brand_context_id#14929523, supplier_id#14929524, canonical_id#14929531, is_primary#14929532, entity_name#14929525, city#14929526, state#14929527, country#14929528, threshold#14929533, create_date#14929534], [isnotnull(canonical_id#14929531)]
+- InMemoryRelation [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, canonical_id#14929531, is_primary#14929532, threshold#14929533, create_date#14929534], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14929523,supplier_id#14929524,entity_name#14929525,city#14929526,state#14929527,country#14929528,supplier_type#14929529,declaring_supplier_id#14929530,canonical_id#14929531,is_primary#14929532,threshold#14929533,create_date#14929534] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(3) InMemoryTableScan
Output [5]: [brand_context_id#14929523, supplier_id#14929524, supplier_type#14929529, declaring_supplier_id#14929530, threshold#14929533]
Arguments: [brand_context_id#14929523, supplier_id#14929524, supplier_type#14929529, declaring_supplier_id#14929530, threshold#14929533], [isnotnull(brand_context_id#14929523), isnotnull(supplier_id#14929524)]
(4) InMemoryRelation
Arguments: [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, threshold#14929533, create_date#14929534], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@730ae8bf,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Project [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, threshold#14929533, create_date#14929534]
+- *(1) Filter isnull(canonical_id#14929531)
+- TableCacheQueryStage 0
+- InMemoryTableScan [brand_context_id#14929523, canonical_id#14929531, city#14929526, country#14929528, create_date#14929534, declaring_supplier_id#14929530, entity_name#14929525, state#14929527, supplier_id#14929524, supplier_type#14929529, threshold#14929533], [isnull(canonical_id#14929531)]
+- InMemoryRelation [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, canonical_id#14929531, is_primary#14929532, threshold#14929533, create_date#14929534], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14929523,supplier_id#14929524,entity_name#14929525,city#14929526,state#14929527,country#14929528,supplier_type#14929529,declaring_supplier_id#14929530,canonical_id#14929531,is_primary#14929532,threshold#14929533,create_date#14929534] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Project [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, threshold#14929533, create_date#14929534]
+- Filter isnull(canonical_id#14929531)
+- InMemoryTableScan [brand_context_id#14929523, canonical_id#14929531, city#14929526, country#14929528, create_date#14929534, declaring_supplier_id#14929530, entity_name#14929525, state#14929527, supplier_id#14929524, supplier_type#14929529, threshold#14929533], [isnull(canonical_id#14929531)]
+- InMemoryRelation [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, canonical_id#14929531, is_primary#14929532, threshold#14929533, create_date#14929534], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14929523,supplier_id#14929524,entity_name#14929525,city#14929526,state#14929527,country#14929528,supplier_type#14929529,declaring_supplier_id#14929530,canonical_id#14929531,is_primary#14929532,threshold#14929533,create_date#14929534] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(5) InMemoryTableScan
Output [11]: [brand_context_id#14929523, canonical_id#14929531, city#14929526, country#14929528, create_date#14929534, declaring_supplier_id#14929530, entity_name#14929525, state#14929527, supplier_id#14929524, supplier_type#14929529, threshold#14929533]
Arguments: [brand_context_id#14929523, canonical_id#14929531, city#14929526, country#14929528, create_date#14929534, declaring_supplier_id#14929530, entity_name#14929525, state#14929527, supplier_id#14929524, supplier_type#14929529, threshold#14929533], [isnull(canonical_id#14929531)]
(6) InMemoryRelation
Arguments: [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, canonical_id#14929531, is_primary#14929532, threshold#14929533, create_date#14929534], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@730ae8bf,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14929523,supplier_id#14929524,entity_name#14929525,city#14929526,state#14929527,country#14929528,supplier_type#14929529,declaring_supplier_id#14929530,canonical_id#14929531,is_primary#14929532,threshold#14929533,create_date#14929534] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(7) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [codegen id : 1]
Output [12]: [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, canonical_id#14929531, is_primary#14929532, threshold#14929533, create_date#14929534]
ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,country:string,supplier_type:string,declaring_supplier_id:string,canonical_id:string,is_primary:boolean,threshold:double,create_date:date>
(8) Filter
Input [11]: [brand_context_id#14929523, canonical_id#14929531, city#14929526, country#14929528, create_date#14929534, declaring_supplier_id#14929530, entity_name#14929525, state#14929527, supplier_id#14929524, supplier_type#14929529, threshold#14929533]
Condition : isnull(canonical_id#14929531)
(9) Project
Output [10]: [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, threshold#14929533, create_date#14929534]
Input [11]: [brand_context_id#14929523, canonical_id#14929531, city#14929526, country#14929528, create_date#14929534, declaring_supplier_id#14929530, entity_name#14929525, state#14929527, supplier_id#14929524, supplier_type#14929529, threshold#14929533]
(10) AdaptiveSparkPlan
Output [10]: [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, threshold#14929533, create_date#14929534]
Arguments: isFinalPlan=false
(11) Filter
Input [5]: [brand_context_id#14929523, supplier_id#14929524, supplier_type#14929529, declaring_supplier_id#14929530, threshold#14929533]
Condition : (isnotnull(brand_context_id#14929523) AND isnotnull(supplier_id#14929524))
(12) BroadcastExchange
Input [5]: [brand_context_id#14929523, supplier_id#14929524, supplier_type#14929529, declaring_supplier_id#14929530, threshold#14929533]
Arguments: HashedRelationBroadcastMode(List(input[0, string, false], input[1, string, false]),false), [plan_id=4291222]
(13) InMemoryTableScan
Output [4]: [brand_context_id#14931472, canonical_id#14931480, is_primary#14931481, supplier_id#14931473]
Arguments: [brand_context_id#14931472, canonical_id#14931480, is_primary#14931481, supplier_id#14931473], [isnotnull(is_primary#14931481), is_primary#14931481, isnotnull(canonical_id#14931480), isnotnull(brand_context_id#14931472), isnotnull(supplier_id#14931473)]
(14) InMemoryRelation
Arguments: [brand_context_id#14931472, supplier_id#14931473, canonical_id#14931480, is_primary#14931481, entity_name#14931474, city#14931475, state#14931476, country#14931477, threshold#14931482, create_date#14931483], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@730ae8bf,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Filter isnotnull(canonical_id#14929531)
+- TableCacheQueryStage 0
+- InMemoryTableScan [brand_context_id#14929523, supplier_id#14929524, canonical_id#14929531, is_primary#14929532, entity_name#14929525, city#14929526, state#14929527, country#14929528, threshold#14929533, create_date#14929534], [isnotnull(canonical_id#14929531)]
+- InMemoryRelation [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, canonical_id#14929531, is_primary#14929532, threshold#14929533, create_date#14929534], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14929523,supplier_id#14929524,entity_name#14929525,city#14929526,state#14929527,country#14929528,supplier_type#14929529,declaring_supplier_id#14929530,canonical_id#14929531,is_primary#14929532,threshold#14929533,create_date#14929534] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Filter isnotnull(canonical_id#14929531)
+- InMemoryTableScan [brand_context_id#14929523, supplier_id#14929524, canonical_id#14929531, is_primary#14929532, entity_name#14929525, city#14929526, state#14929527, country#14929528, threshold#14929533, create_date#14929534], [isnotnull(canonical_id#14929531)]
+- InMemoryRelation [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, canonical_id#14929531, is_primary#14929532, threshold#14929533, create_date#14929534], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14929523,supplier_id#14929524,entity_name#14929525,city#14929526,state#14929527,country#14929528,supplier_type#14929529,declaring_supplier_id#14929530,canonical_id#14929531,is_primary#14929532,threshold#14929533,create_date#14929534] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(15) InMemoryTableScan
Output [10]: [brand_context_id#14929523, supplier_id#14929524, canonical_id#14929531, is_primary#14929532, entity_name#14929525, city#14929526, state#14929527, country#14929528, threshold#14929533, create_date#14929534]
Arguments: [brand_context_id#14929523, supplier_id#14929524, canonical_id#14929531, is_primary#14929532, entity_name#14929525, city#14929526, state#14929527, country#14929528, threshold#14929533, create_date#14929534], [isnotnull(canonical_id#14929531)]
(16) InMemoryRelation
Arguments: [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, canonical_id#14929531, is_primary#14929532, threshold#14929533, create_date#14929534], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@730ae8bf,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14929523,supplier_id#14929524,entity_name#14929525,city#14929526,state#14929527,country#14929528,supplier_type#14929529,declaring_supplier_id#14929530,canonical_id#14929531,is_primary#14929532,threshold#14929533,create_date#14929534] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(17) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [codegen id : 1]
Output [12]: [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, canonical_id#14929531, is_primary#14929532, threshold#14929533, create_date#14929534]
ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,country:string,supplier_type:string,declaring_supplier_id:string,canonical_id:string,is_primary:boolean,threshold:double,create_date:date>
(18) Filter
Input [10]: [brand_context_id#14929523, supplier_id#14929524, canonical_id#14929531, is_primary#14929532, entity_name#14929525, city#14929526, state#14929527, country#14929528, threshold#14929533, create_date#14929534]
Condition : isnotnull(canonical_id#14929531)
(19) AdaptiveSparkPlan
Output [10]: [brand_context_id#14929523, supplier_id#14929524, canonical_id#14929531, is_primary#14929532, entity_name#14929525, city#14929526, state#14929527, country#14929528, threshold#14929533, create_date#14929534]
Arguments: isFinalPlan=false
(20) Filter
Input [4]: [brand_context_id#14931472, canonical_id#14931480, is_primary#14931481, supplier_id#14931473]
Condition : ((((isnotnull(is_primary#14931481) AND is_primary#14931481) AND isnotnull(canonical_id#14931480)) AND isnotnull(brand_context_id#14931472)) AND isnotnull(supplier_id#14931473))
(21) Project
Output [3]: [brand_context_id#14931472, supplier_id#14931473, canonical_id#14931480]
Input [4]: [brand_context_id#14931472, canonical_id#14931480, is_primary#14931481, supplier_id#14931473]
(22) BroadcastHashJoin
Left keys [2]: [brand_context_id#14929523, supplier_id#14929524]
Right keys [2]: [brand_context_id#14931472, supplier_id#14931473]
Join type: Inner
Join condition: None
(23) Project
Output [10]: [brand_context_id#14929523, supplier_id#14929524, declaring_supplier_id#14929530, supplier_type#14929529, canonical_id#14931480, 0 AS is_primary#14931544, threshold#14929533 AS configured_threshold#14931545, 100.0 AS match_percentage#14931546, 1.0 AS confidence_score#14931547, exact_id_match AS match_reason#14931548]
Input [8]: [brand_context_id#14929523, supplier_id#14929524, supplier_type#14929529, declaring_supplier_id#14929530, threshold#14929533, brand_context_id#14931472, supplier_id#14931473, canonical_id#14931480]
(24) InMemoryTableScan
Output [9]: [brand_context_id#14931798, supplier_id#14931799, entity_name#14931800, city#14931801, state#14931802, country#14931803, supplier_type#14931804, declaring_supplier_id#14931805, threshold#14931808]
Arguments: [brand_context_id#14931798, supplier_id#14931799, entity_name#14931800, city#14931801, state#14931802, country#14931803, supplier_type#14931804, declaring_supplier_id#14931805, threshold#14931808], [isnotnull(threshold#14931808), isnotnull(brand_context_id#14931798)]
(25) InMemoryRelation
Arguments: [brand_context_id#14931798, supplier_id#14931799, entity_name#14931800, city#14931801, state#14931802, country#14931803, supplier_type#14931804, declaring_supplier_id#14931805, threshold#14931808, create_date#14931809], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@730ae8bf,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Project [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, threshold#14929533, create_date#14929534]
+- *(1) Filter isnull(canonical_id#14929531)
+- TableCacheQueryStage 0
+- InMemoryTableScan [brand_context_id#14929523, canonical_id#14929531, city#14929526, country#14929528, create_date#14929534, declaring_supplier_id#14929530, entity_name#14929525, state#14929527, supplier_id#14929524, supplier_type#14929529, threshold#14929533], [isnull(canonical_id#14929531)]
+- InMemoryRelation [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, canonical_id#14929531, is_primary#14929532, threshold#14929533, create_date#14929534], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14929523,supplier_id#14929524,entity_name#14929525,city#14929526,state#14929527,country#14929528,supplier_type#14929529,declaring_supplier_id#14929530,canonical_id#14929531,is_primary#14929532,threshold#14929533,create_date#14929534] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Project [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, threshold#14929533, create_date#14929534]
+- Filter isnull(canonical_id#14929531)
+- InMemoryTableScan [brand_context_id#14929523, canonical_id#14929531, city#14929526, country#14929528, create_date#14929534, declaring_supplier_id#14929530, entity_name#14929525, state#14929527, supplier_id#14929524, supplier_type#14929529, threshold#14929533], [isnull(canonical_id#14929531)]
+- InMemoryRelation [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, canonical_id#14929531, is_primary#14929532, threshold#14929533, create_date#14929534], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14929523,supplier_id#14929524,entity_name#14929525,city#14929526,state#14929527,country#14929528,supplier_type#14929529,declaring_supplier_id#14929530,canonical_id#14929531,is_primary#14929532,threshold#14929533,create_date#14929534] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(26) AdaptiveSparkPlan
Output [10]: [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, threshold#14929533, create_date#14929534]
Arguments: isFinalPlan=false
(27) Filter
Input [9]: [brand_context_id#14931798, supplier_id#14931799, entity_name#14931800, city#14931801, state#14931802, country#14931803, supplier_type#14931804, declaring_supplier_id#14931805, threshold#14931808]
Condition : (isnotnull(threshold#14931808) AND isnotnull(brand_context_id#14931798))
(28) BatchEvalPython
Input [9]: [brand_context_id#14931798, supplier_id#14931799, entity_name#14931800, city#14931801, state#14931802, country#14931803, supplier_type#14931804, declaring_supplier_id#14931805, threshold#14931808]
Arguments: [normalize_location(city#14931801)#14931067, normalize_location(state#14931802)#14931081, normalize_location(country#14931803)#14931096], [pythonUDF0#14933192, pythonUDF1#14933193, pythonUDF2#14933194]
(29) Filter
Input [12]: [brand_context_id#14931798, supplier_id#14931799, entity_name#14931800, city#14931801, state#14931802, country#14931803, supplier_type#14931804, declaring_supplier_id#14931805, threshold#14931808, pythonUDF0#14933192, pythonUDF1#14933193, pythonUDF2#14933194]
Condition : ((isnotnull(pythonUDF0#14933192) AND isnotnull(pythonUDF1#14933193)) AND isnotnull(pythonUDF2#14933194))
(30) Project
Output [9]: [brand_context_id#14931798, supplier_id#14931799, entity_name#14931800, city#14931801, state#14931802, country#14931803, supplier_type#14931804, declaring_supplier_id#14931805, threshold#14931808]
Input [12]: [brand_context_id#14931798, supplier_id#14931799, entity_name#14931800, city#14931801, state#14931802, country#14931803, supplier_type#14931804, declaring_supplier_id#14931805, threshold#14931808, pythonUDF0#14933192, pythonUDF1#14933193, pythonUDF2#14933194]
(31) BatchEvalPython
Input [9]: [brand_context_id#14931798, supplier_id#14931799, entity_name#14931800, city#14931801, state#14931802, country#14931803, supplier_type#14931804, declaring_supplier_id#14931805, threshold#14931808]
Arguments: [normalize_name(entity_name#14931800)#14931054, normalize_location(city#14931801)#14931067, normalize_location(state#14931802)#14931081, normalize_location(country#14931803)#14931096], [pythonUDF0#14933195, pythonUDF1#14933196, pythonUDF2#14933197, pythonUDF3#14933198]
(32) Project
Output [7]: [brand_context_id#14931798, supplier_id#14931799, supplier_type#14931804, declaring_supplier_id#14931805, threshold#14931808, pythonUDF0#14933195 AS normalized_name#14931055, concat_ws(|, coalesce(pythonUDF1#14933196, ), coalesce(pythonUDF2#14933197, ), coalesce(pythonUDF3#14933198, )) AS location_key#14931112]
Input [13]: [brand_context_id#14931798, supplier_id#14931799, entity_name#14931800, city#14931801, state#14931802, country#14931803, supplier_type#14931804, declaring_supplier_id#14931805, threshold#14931808, pythonUDF0#14933195, pythonUDF1#14933196, pythonUDF2#14933197, pythonUDF3#14933198]
(33) Exchange
Input [7]: [brand_context_id#14931798, supplier_id#14931799, supplier_type#14931804, declaring_supplier_id#14931805, threshold#14931808, normalized_name#14931055, location_key#14931112]
Arguments: hashpartitioning(brand_context_id#14931798, supplier_id#14931799, 10), ENSURE_REQUIREMENTS, [plan_id=4291230]
(34) Sort
Input [7]: [brand_context_id#14931798, supplier_id#14931799, supplier_type#14931804, declaring_supplier_id#14931805, threshold#14931808, normalized_name#14931055, location_key#14931112]
Arguments: [brand_context_id#14931798 ASC NULLS FIRST, supplier_id#14931799 ASC NULLS FIRST], false, 0
(35) InMemoryTableScan
Output [2]: [brand_context_id#14931561, supplier_id#14931562]
Arguments: [brand_context_id#14931561, supplier_id#14931562], [isnotnull(brand_context_id#14931561), isnotnull(supplier_id#14931562)]
(36) InMemoryRelation
Arguments: [brand_context_id#14931561, supplier_id#14931562, entity_name#14931563, city#14931564, state#14931565, country#14931566, supplier_type#14931567, declaring_supplier_id#14931568, threshold#14931571, create_date#14931572], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@730ae8bf,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Project [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, threshold#14929533, create_date#14929534]
+- *(1) Filter isnull(canonical_id#14929531)
+- TableCacheQueryStage 0
+- InMemoryTableScan [brand_context_id#14929523, canonical_id#14929531, city#14929526, country#14929528, create_date#14929534, declaring_supplier_id#14929530, entity_name#14929525, state#14929527, supplier_id#14929524, supplier_type#14929529, threshold#14929533], [isnull(canonical_id#14929531)]
+- InMemoryRelation [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, canonical_id#14929531, is_primary#14929532, threshold#14929533, create_date#14929534], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14929523,supplier_id#14929524,entity_name#14929525,city#14929526,state#14929527,country#14929528,supplier_type#14929529,declaring_supplier_id#14929530,canonical_id#14929531,is_primary#14929532,threshold#14929533,create_date#14929534] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Project [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, threshold#14929533, create_date#14929534]
+- Filter isnull(canonical_id#14929531)
+- InMemoryTableScan [brand_context_id#14929523, canonical_id#14929531, city#14929526, country#14929528, create_date#14929534, declaring_supplier_id#14929530, entity_name#14929525, state#14929527, supplier_id#14929524, supplier_type#14929529, threshold#14929533], [isnull(canonical_id#14929531)]
+- InMemoryRelation [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, canonical_id#14929531, is_primary#14929532, threshold#14929533, create_date#14929534], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14929523,supplier_id#14929524,entity_name#14929525,city#14929526,state#14929527,country#14929528,supplier_type#14929529,declaring_supplier_id#14929530,canonical_id#14929531,is_primary#14929532,threshold#14929533,create_date#14929534] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(37) AdaptiveSparkPlan
Output [10]: [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, threshold#14929533, create_date#14929534]
Arguments: isFinalPlan=false
(38) Filter
Input [2]: [brand_context_id#14931561, supplier_id#14931562]
Condition : (isnotnull(brand_context_id#14931561) AND isnotnull(supplier_id#14931562))
(39) BroadcastExchange
Input [2]: [brand_context_id#14931561, supplier_id#14931562]
Arguments: HashedRelationBroadcastMode(List(input[0, string, false], input[1, string, false]),false), [plan_id=4291225]
(40) InMemoryTableScan
Output [4]: [brand_context_id#14931810, canonical_id#14931818, is_primary#14931819, supplier_id#14931811]
Arguments: [brand_context_id#14931810, canonical_id#14931818, is_primary#14931819, supplier_id#14931811], [isnotnull(is_primary#14931819), is_primary#14931819, isnotnull(canonical_id#14931818), isnotnull(brand_context_id#14931810), isnotnull(supplier_id#14931811)]
(41) InMemoryRelation
Arguments: [brand_context_id#14931810, supplier_id#14931811, canonical_id#14931818, is_primary#14931819, entity_name#14931812, city#14931813, state#14931814, country#14931815, threshold#14931820, create_date#14931821], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@730ae8bf,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Filter isnotnull(canonical_id#14929531)
+- TableCacheQueryStage 0
+- InMemoryTableScan [brand_context_id#14929523, supplier_id#14929524, canonical_id#14929531, is_primary#14929532, entity_name#14929525, city#14929526, state#14929527, country#14929528, threshold#14929533, create_date#14929534], [isnotnull(canonical_id#14929531)]
+- InMemoryRelation [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, canonical_id#14929531, is_primary#14929532, threshold#14929533, create_date#14929534], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14929523,supplier_id#14929524,entity_name#14929525,city#14929526,state#14929527,country#14929528,supplier_type#14929529,declaring_supplier_id#14929530,canonical_id#14929531,is_primary#14929532,threshold#14929533,create_date#14929534] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Filter isnotnull(canonical_id#14929531)
+- InMemoryTableScan [brand_context_id#14929523, supplier_id#14929524, canonical_id#14929531, is_primary#14929532, entity_name#14929525, city#14929526, state#14929527, country#14929528, threshold#14929533, create_date#14929534], [isnotnull(canonical_id#14929531)]
+- InMemoryRelation [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, canonical_id#14929531, is_primary#14929532, threshold#14929533, create_date#14929534], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14929523,supplier_id#14929524,entity_name#14929525,city#14929526,state#14929527,country#14929528,supplier_type#14929529,declaring_supplier_id#14929530,canonical_id#14929531,is_primary#14929532,threshold#14929533,create_date#14929534] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(42) AdaptiveSparkPlan
Output [10]: [brand_context_id#14929523, supplier_id#14929524, canonical_id#14929531, is_primary#14929532, entity_name#14929525, city#14929526, state#14929527, country#14929528, threshold#14929533, create_date#14929534]
Arguments: isFinalPlan=false
(43) Filter
Input [4]: [brand_context_id#14931810, canonical_id#14931818, is_primary#14931819, supplier_id#14931811]
Condition : ((((isnotnull(is_primary#14931819) AND is_primary#14931819) AND isnotnull(canonical_id#14931818)) AND isnotnull(brand_context_id#14931810)) AND isnotnull(supplier_id#14931811))
(44) Project
Output [2]: [brand_context_id#14931810, supplier_id#14931811]
Input [4]: [brand_context_id#14931810, canonical_id#14931818, is_primary#14931819, supplier_id#14931811]
(45) BroadcastHashJoin
Left keys [2]: [brand_context_id#14931561, supplier_id#14931562]
Right keys [2]: [brand_context_id#14931810, supplier_id#14931811]
Join type: Inner
Join condition: None
(46) Project
Output [2]: [brand_context_id#14931561, supplier_id#14931562]
Input [4]: [brand_context_id#14931561, supplier_id#14931562, brand_context_id#14931810, supplier_id#14931811]
(47) Exchange
Input [2]: [brand_context_id#14931561, supplier_id#14931562]
Arguments: hashpartitioning(brand_context_id#14931561, supplier_id#14931562, 10), ENSURE_REQUIREMENTS, [plan_id=4291231]
(48) Sort
Input [2]: [brand_context_id#14931561, supplier_id#14931562]
Arguments: [brand_context_id#14931561 ASC NULLS FIRST, supplier_id#14931562 ASC NULLS FIRST], false, 0
(49) SortMergeJoin
Left keys [2]: [brand_context_id#14931798, supplier_id#14931799]
Right keys [2]: [brand_context_id#14931561, supplier_id#14931562]
Join type: LeftAnti
Join condition: None
(50) Project
Output [7]: [brand_context_id#14931798 AS u_brand_context_id#14931588, supplier_id#14931799 AS u_supplier_id#14931589, declaring_supplier_id#14931805 AS u_declaring_supplier_id#14931590, supplier_type#14931804 AS u_supplier_type#14931591, threshold#14931808 AS u_threshold#14931592, normalized_name#14931055 AS u_normalized_name#14931593, location_key#14931112 AS u_location_key#14931597]
Input [7]: [brand_context_id#14931798, supplier_id#14931799, supplier_type#14931804, declaring_supplier_id#14931805, threshold#14931808, normalized_name#14931055, location_key#14931112]
(51) BroadcastExchange
Input [7]: [u_brand_context_id#14931588, u_supplier_id#14931589, u_declaring_supplier_id#14931590, u_supplier_type#14931591, u_threshold#14931592, u_normalized_name#14931593, u_location_key#14931597]
Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[6, string, false]),false), [plan_id=4291237]
(52) InMemoryTableScan
Output [8]: [brand_context_id#14931622, supplier_id#14931623, canonical_id#14931630, entity_name#14931624, city#14931625, state#14931626, country#14931627, create_date#14931633]
Arguments: [brand_context_id#14931622, supplier_id#14931623, canonical_id#14931630, entity_name#14931624, city#14931625, state#14931626, country#14931627, create_date#14931633], [isnotnull(brand_context_id#14931622)]
(53) InMemoryRelation
Arguments: [brand_context_id#14931622, supplier_id#14931623, canonical_id#14931630, is_primary#14931631, entity_name#14931624, city#14931625, state#14931626, country#14931627, threshold#14931632, create_date#14931633], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@730ae8bf,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Filter isnotnull(canonical_id#14929531)
+- TableCacheQueryStage 0
+- InMemoryTableScan [brand_context_id#14929523, supplier_id#14929524, canonical_id#14929531, is_primary#14929532, entity_name#14929525, city#14929526, state#14929527, country#14929528, threshold#14929533, create_date#14929534], [isnotnull(canonical_id#14929531)]
+- InMemoryRelation [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, canonical_id#14929531, is_primary#14929532, threshold#14929533, create_date#14929534], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14929523,supplier_id#14929524,entity_name#14929525,city#14929526,state#14929527,country#14929528,supplier_type#14929529,declaring_supplier_id#14929530,canonical_id#14929531,is_primary#14929532,threshold#14929533,create_date#14929534] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Filter isnotnull(canonical_id#14929531)
+- InMemoryTableScan [brand_context_id#14929523, supplier_id#14929524, canonical_id#14929531, is_primary#14929532, entity_name#14929525, city#14929526, state#14929527, country#14929528, threshold#14929533, create_date#14929534], [isnotnull(canonical_id#14929531)]
+- InMemoryRelation [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, canonical_id#14929531, is_primary#14929532, threshold#14929533, create_date#14929534], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14929523,supplier_id#14929524,entity_name#14929525,city#14929526,state#14929527,country#14929528,supplier_type#14929529,declaring_supplier_id#14929530,canonical_id#14929531,is_primary#14929532,threshold#14929533,create_date#14929534] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(54) AdaptiveSparkPlan
Output [10]: [brand_context_id#14929523, supplier_id#14929524, canonical_id#14929531, is_primary#14929532, entity_name#14929525, city#14929526, state#14929527, country#14929528, threshold#14929533, create_date#14929534]
Arguments: isFinalPlan=false
(55) Filter
Input [8]: [brand_context_id#14931622, supplier_id#14931623, canonical_id#14931630, entity_name#14931624, city#14931625, state#14931626, country#14931627, create_date#14931633]
Condition : isnotnull(brand_context_id#14931622)
(56) BatchEvalPython
Input [8]: [brand_context_id#14931622, supplier_id#14931623, canonical_id#14931630, entity_name#14931624, city#14931625, state#14931626, country#14931627, create_date#14931633]
Arguments: [normalize_name(entity_name#14931624)#14931128, normalize_location(city#14931625)#14931141, normalize_location(state#14931626)#14931155, normalize_location(country#14931627)#14931170], [pythonUDF0#14933199, pythonUDF1#14933200, pythonUDF2#14933201, pythonUDF3#14933202]
(57) Project
Output [6]: [brand_context_id#14931622 AS e_brand_context_id#14931610, supplier_id#14931623 AS e_supplier_id#14931611, canonical_id#14931630 AS e_canonical_id#14931612, pythonUDF0#14933199 AS e_normalized_name#14931613, concat_ws(|, coalesce(pythonUDF1#14933200, ), coalesce(pythonUDF2#14933201, ), coalesce(pythonUDF3#14933202, )) AS e_location_key#14931614, create_date#14931633 AS e_create_date#14931615]
Input [12]: [brand_context_id#14931622, supplier_id#14931623, canonical_id#14931630, entity_name#14931624, city#14931625, state#14931626, country#14931627, create_date#14931633, pythonUDF0#14933199, pythonUDF1#14933200, pythonUDF2#14933201, pythonUDF3#14933202]
(58) BroadcastHashJoin
Left keys [2]: [u_brand_context_id#14931588, u_location_key#14931597]
Right keys [2]: [e_brand_context_id#14931610, e_location_key#14931614]
Join type: Inner
Join condition: None
(59) Project
Output [10]: [u_brand_context_id#14931588, u_supplier_id#14931589, u_declaring_supplier_id#14931590, u_supplier_type#14931591, u_threshold#14931592, u_normalized_name#14931593, e_supplier_id#14931611, e_canonical_id#14931612, e_normalized_name#14931613, e_create_date#14931615]
Input [13]: [u_brand_context_id#14931588, u_supplier_id#14931589, u_declaring_supplier_id#14931590, u_supplier_type#14931591, u_threshold#14931592, u_normalized_name#14931593, u_location_key#14931597, e_brand_context_id#14931610, e_supplier_id#14931611, e_canonical_id#14931612, e_normalized_name#14931613, e_location_key#14931614, e_create_date#14931615]
(60) BatchEvalPython
Input [10]: [u_brand_context_id#14931588, u_supplier_id#14931589, u_declaring_supplier_id#14931590, u_supplier_type#14931591, u_threshold#14931592, u_normalized_name#14931593, e_supplier_id#14931611, e_canonical_id#14931612, e_normalized_name#14931613, e_create_date#14931615]
Arguments: [calculate_similarity(u_normalized_name#14931593, e_normalized_name#14931613)#14931668], [pythonUDF0#14933203]
(61) Filter
Input [11]: [u_brand_context_id#14931588, u_supplier_id#14931589, u_declaring_supplier_id#14931590, u_supplier_type#14931591, u_threshold#14931592, u_normalized_name#14931593, e_supplier_id#14931611, e_canonical_id#14931612, e_normalized_name#14931613, e_create_date#14931615, pythonUDF0#14933203]
Condition : (cast(pythonUDF0#14933203 as double) >= u_threshold#14931592)
(62) Project
Output [10]: [u_brand_context_id#14931588, u_supplier_id#14931589, u_declaring_supplier_id#14931590, u_supplier_type#14931591, u_threshold#14931592, u_normalized_name#14931593, e_supplier_id#14931611, e_canonical_id#14931612, e_normalized_name#14931613, e_create_date#14931615]
Input [11]: [u_brand_context_id#14931588, u_supplier_id#14931589, u_declaring_supplier_id#14931590, u_supplier_type#14931591, u_threshold#14931592, u_normalized_name#14931593, e_supplier_id#14931611, e_canonical_id#14931612, e_normalized_name#14931613, e_create_date#14931615, pythonUDF0#14933203]
(63) BatchEvalPython
Input [10]: [u_brand_context_id#14931588, u_supplier_id#14931589, u_declaring_supplier_id#14931590, u_supplier_type#14931591, u_threshold#14931592, u_normalized_name#14931593, e_supplier_id#14931611, e_canonical_id#14931612, e_normalized_name#14931613, e_create_date#14931615]
Arguments: [calculate_similarity(u_normalized_name#14931593, e_normalized_name#14931613)#14931668], [pythonUDF0#14933204]
(64) Project
Output [9]: [u_brand_context_id#14931588, u_supplier_id#14931589, u_declaring_supplier_id#14931590, u_supplier_type#14931591, u_threshold#14931592, e_supplier_id#14931611, e_canonical_id#14931612, e_create_date#14931615, pythonUDF0#14933204 AS similarity#14931669]
Input [11]: [u_brand_context_id#14931588, u_supplier_id#14931589, u_declaring_supplier_id#14931590, u_supplier_type#14931591, u_threshold#14931592, u_normalized_name#14931593, e_supplier_id#14931611, e_canonical_id#14931612, e_normalized_name#14931613, e_create_date#14931615, pythonUDF0#14933204]
(65) Sort
Input [9]: [u_brand_context_id#14931588, u_supplier_id#14931589, u_declaring_supplier_id#14931590, u_supplier_type#14931591, u_threshold#14931592, e_supplier_id#14931611, e_canonical_id#14931612, e_create_date#14931615, similarity#14931669]
Arguments: [u_brand_context_id#14931588 ASC NULLS FIRST, u_supplier_id#14931589 ASC NULLS FIRST, similarity#14931669 DESC NULLS LAST, e_create_date#14931615 ASC NULLS FIRST, e_supplier_id#14931611 ASC NULLS FIRST], false, 0
(66) WindowGroupLimit
Input [9]: [u_brand_context_id#14931588, u_supplier_id#14931589, u_declaring_supplier_id#14931590, u_supplier_type#14931591, u_threshold#14931592, e_supplier_id#14931611, e_canonical_id#14931612, e_create_date#14931615, similarity#14931669]
Arguments: [u_brand_context_id#14931588, u_supplier_id#14931589], [similarity#14931669 DESC NULLS LAST, e_create_date#14931615 ASC NULLS FIRST, e_supplier_id#14931611 ASC NULLS FIRST], row_number(), 1, Partial
(67) Exchange
Input [9]: [u_brand_context_id#14931588, u_supplier_id#14931589, u_declaring_supplier_id#14931590, u_supplier_type#14931591, u_threshold#14931592, e_supplier_id#14931611, e_canonical_id#14931612, e_create_date#14931615, similarity#14931669]
Arguments: hashpartitioning(u_brand_context_id#14931588, u_supplier_id#14931589, 10), ENSURE_REQUIREMENTS, [plan_id=4291249]
(68) Sort
Input [9]: [u_brand_context_id#14931588, u_supplier_id#14931589, u_declaring_supplier_id#14931590, u_supplier_type#14931591, u_threshold#14931592, e_supplier_id#14931611, e_canonical_id#14931612, e_create_date#14931615, similarity#14931669]
Arguments: [u_brand_context_id#14931588 ASC NULLS FIRST, u_supplier_id#14931589 ASC NULLS FIRST, similarity#14931669 DESC NULLS LAST, e_create_date#14931615 ASC NULLS FIRST, e_supplier_id#14931611 ASC NULLS FIRST], false, 0
(69) WindowGroupLimit
Input [9]: [u_brand_context_id#14931588, u_supplier_id#14931589, u_declaring_supplier_id#14931590, u_supplier_type#14931591, u_threshold#14931592, e_supplier_id#14931611, e_canonical_id#14931612, e_create_date#14931615, similarity#14931669]
Arguments: [u_brand_context_id#14931588, u_supplier_id#14931589], [similarity#14931669 DESC NULLS LAST, e_create_date#14931615 ASC NULLS FIRST, e_supplier_id#14931611 ASC NULLS FIRST], row_number(), 1, Final
(70) Window
Input [9]: [u_brand_context_id#14931588, u_supplier_id#14931589, u_declaring_supplier_id#14931590, u_supplier_type#14931591, u_threshold#14931592, e_supplier_id#14931611, e_canonical_id#14931612, e_create_date#14931615, similarity#14931669]
Arguments: [row_number() windowspecdefinition(u_brand_context_id#14931588, u_supplier_id#14931589, similarity#14931669 DESC NULLS LAST, e_create_date#14931615 ASC NULLS FIRST, e_supplier_id#14931611 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank#14931689], [u_brand_context_id#14931588, u_supplier_id#14931589], [similarity#14931669 DESC NULLS LAST, e_create_date#14931615 ASC NULLS FIRST, e_supplier_id#14931611 ASC NULLS FIRST]
(71) Filter
Input [10]: [u_brand_context_id#14931588, u_supplier_id#14931589, u_declaring_supplier_id#14931590, u_supplier_type#14931591, u_threshold#14931592, e_supplier_id#14931611, e_canonical_id#14931612, e_create_date#14931615, similarity#14931669, rank#14931689]
Condition : (rank#14931689 = 1)
(72) Project
Output [10]: [u_brand_context_id#14931588 AS brand_context_id#14931709, u_supplier_id#14931589 AS supplier_id#14931710, u_declaring_supplier_id#14931590 AS declaring_supplier_id#14931711, u_supplier_type#14931591 AS supplier_type#14931712, e_canonical_id#14931612 AS canonical_id#14931713, 0 AS is_primary#14931714, u_threshold#14931592 AS configured_threshold#14931715, cast((similarity#14931669 * 100.0) as double) AS match_percentage#14931822, cast(similarity#14931669 as double) AS confidence_score#14931823, matched_existing AS match_reason#14931718]
Input [10]: [u_brand_context_id#14931588, u_supplier_id#14931589, u_declaring_supplier_id#14931590, u_supplier_type#14931591, u_threshold#14931592, e_supplier_id#14931611, e_canonical_id#14931612, e_create_date#14931615, similarity#14931669, rank#14931689]
(73) InMemoryTableScan
Output [5]: [brand_context_id#14931834, supplier_id#14931835, supplier_type#14931840, declaring_supplier_id#14931841, threshold#14931844]
Arguments: [brand_context_id#14931834, supplier_id#14931835, supplier_type#14931840, declaring_supplier_id#14931841, threshold#14931844]
(74) InMemoryRelation
Arguments: [brand_context_id#14931834, supplier_id#14931835, entity_name#14931836, city#14931837, state#14931838, country#14931839, supplier_type#14931840, declaring_supplier_id#14931841, threshold#14931844, create_date#14931845], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@730ae8bf,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Project [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, threshold#14929533, create_date#14929534]
+- *(1) Filter isnull(canonical_id#14929531)
+- TableCacheQueryStage 0
+- InMemoryTableScan [brand_context_id#14929523, canonical_id#14929531, city#14929526, country#14929528, create_date#14929534, declaring_supplier_id#14929530, entity_name#14929525, state#14929527, supplier_id#14929524, supplier_type#14929529, threshold#14929533], [isnull(canonical_id#14929531)]
+- InMemoryRelation [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, canonical_id#14929531, is_primary#14929532, threshold#14929533, create_date#14929534], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14929523,supplier_id#14929524,entity_name#14929525,city#14929526,state#14929527,country#14929528,supplier_type#14929529,declaring_supplier_id#14929530,canonical_id#14929531,is_primary#14929532,threshold#14929533,create_date#14929534] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Project [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, threshold#14929533, create_date#14929534]
+- Filter isnull(canonical_id#14929531)
+- InMemoryTableScan [brand_context_id#14929523, canonical_id#14929531, city#14929526, country#14929528, create_date#14929534, declaring_supplier_id#14929530, entity_name#14929525, state#14929527, supplier_id#14929524, supplier_type#14929529, threshold#14929533], [isnull(canonical_id#14929531)]
+- InMemoryRelation [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, canonical_id#14929531, is_primary#14929532, threshold#14929533, create_date#14929534], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14929523,supplier_id#14929524,entity_name#14929525,city#14929526,state#14929527,country#14929528,supplier_type#14929529,declaring_supplier_id#14929530,canonical_id#14929531,is_primary#14929532,threshold#14929533,create_date#14929534] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(75) AdaptiveSparkPlan
Output [10]: [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, threshold#14929533, create_date#14929534]
Arguments: isFinalPlan=false
(76) Exchange
Input [5]: [brand_context_id#14931834, supplier_id#14931835, supplier_type#14931840, declaring_supplier_id#14931841, threshold#14931844]
Arguments: hashpartitioning(brand_context_id#14931834, supplier_id#14931835, 10), ENSURE_REQUIREMENTS, [plan_id=4291260]
(77) Sort
Input [5]: [brand_context_id#14931834, supplier_id#14931835, supplier_type#14931840, declaring_supplier_id#14931841, threshold#14931844]
Arguments: [brand_context_id#14931834 ASC NULLS FIRST, supplier_id#14931835 ASC NULLS FIRST], false, 0
(78) InMemoryTableScan
Output [2]: [brand_context_id#14931846, supplier_id#14931847]
Arguments: [brand_context_id#14931846, supplier_id#14931847], [isnotnull(brand_context_id#14931846), isnotnull(supplier_id#14931847)]
(79) InMemoryRelation
Arguments: [brand_context_id#14931846, supplier_id#14931847, entity_name#14931848, city#14931849, state#14931850, country#14931851, supplier_type#14931852, declaring_supplier_id#14931853, threshold#14931856, create_date#14931857], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@730ae8bf,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Project [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, threshold#14929533, create_date#14929534]
+- *(1) Filter isnull(canonical_id#14929531)
+- TableCacheQueryStage 0
+- InMemoryTableScan [brand_context_id#14929523, canonical_id#14929531, city#14929526, country#14929528, create_date#14929534, declaring_supplier_id#14929530, entity_name#14929525, state#14929527, supplier_id#14929524, supplier_type#14929529, threshold#14929533], [isnull(canonical_id#14929531)]
+- InMemoryRelation [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, canonical_id#14929531, is_primary#14929532, threshold#14929533, create_date#14929534], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14929523,supplier_id#14929524,entity_name#14929525,city#14929526,state#14929527,country#14929528,supplier_type#14929529,declaring_supplier_id#14929530,canonical_id#14929531,is_primary#14929532,threshold#14929533,create_date#14929534] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Project [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, threshold#14929533, create_date#14929534]
+- Filter isnull(canonical_id#14929531)
+- InMemoryTableScan [brand_context_id#14929523, canonical_id#14929531, city#14929526, country#14929528, create_date#14929534, declaring_supplier_id#14929530, entity_name#14929525, state#14929527, supplier_id#14929524, supplier_type#14929529, threshold#14929533], [isnull(canonical_id#14929531)]
+- InMemoryRelation [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, canonical_id#14929531, is_primary#14929532, threshold#14929533, create_date#14929534], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14929523,supplier_id#14929524,entity_name#14929525,city#14929526,state#14929527,country#14929528,supplier_type#14929529,declaring_supplier_id#14929530,canonical_id#14929531,is_primary#14929532,threshold#14929533,create_date#14929534] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(80) AdaptiveSparkPlan
Output [10]: [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, threshold#14929533, create_date#14929534]
Arguments: isFinalPlan=false
(81) Filter
Input [2]: [brand_context_id#14931846, supplier_id#14931847]
Condition : (isnotnull(brand_context_id#14931846) AND isnotnull(supplier_id#14931847))
(82) BroadcastExchange
Input [2]: [brand_context_id#14931846, supplier_id#14931847]
Arguments: HashedRelationBroadcastMode(List(input[0, string, false], input[1, string, false]),false), [plan_id=4291255]
(83) InMemoryTableScan
Output [4]: [brand_context_id#14931858, canonical_id#14931866, is_primary#14931867, supplier_id#14931859]
Arguments: [brand_context_id#14931858, canonical_id#14931866, is_primary#14931867, supplier_id#14931859], [isnotnull(is_primary#14931867), is_primary#14931867, isnotnull(canonical_id#14931866), isnotnull(brand_context_id#14931858), isnotnull(supplier_id#14931859)]
(84) InMemoryRelation
Arguments: [brand_context_id#14931858, supplier_id#14931859, canonical_id#14931866, is_primary#14931867, entity_name#14931860, city#14931861, state#14931862, country#14931863, threshold#14931868, create_date#14931869], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@730ae8bf,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Filter isnotnull(canonical_id#14929531)
+- TableCacheQueryStage 0
+- InMemoryTableScan [brand_context_id#14929523, supplier_id#14929524, canonical_id#14929531, is_primary#14929532, entity_name#14929525, city#14929526, state#14929527, country#14929528, threshold#14929533, create_date#14929534], [isnotnull(canonical_id#14929531)]
+- InMemoryRelation [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, canonical_id#14929531, is_primary#14929532, threshold#14929533, create_date#14929534], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14929523,supplier_id#14929524,entity_name#14929525,city#14929526,state#14929527,country#14929528,supplier_type#14929529,declaring_supplier_id#14929530,canonical_id#14929531,is_primary#14929532,threshold#14929533,create_date#14929534] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Filter isnotnull(canonical_id#14929531)
+- InMemoryTableScan [brand_context_id#14929523, supplier_id#14929524, canonical_id#14929531, is_primary#14929532, entity_name#14929525, city#14929526, state#14929527, country#14929528, threshold#14929533, create_date#14929534], [isnotnull(canonical_id#14929531)]
+- InMemoryRelation [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, canonical_id#14929531, is_primary#14929532, threshold#14929533, create_date#14929534], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14929523,supplier_id#14929524,entity_name#14929525,city#14929526,state#14929527,country#14929528,supplier_type#14929529,declaring_supplier_id#14929530,canonical_id#14929531,is_primary#14929532,threshold#14929533,create_date#14929534] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(85) AdaptiveSparkPlan
Output [10]: [brand_context_id#14929523, supplier_id#14929524, canonical_id#14929531, is_primary#14929532, entity_name#14929525, city#14929526, state#14929527, country#14929528, threshold#14929533, create_date#14929534]
Arguments: isFinalPlan=false
(86) Filter
Input [4]: [brand_context_id#14931858, canonical_id#14931866, is_primary#14931867, supplier_id#14931859]
Condition : ((((isnotnull(is_primary#14931867) AND is_primary#14931867) AND isnotnull(canonical_id#14931866)) AND isnotnull(brand_context_id#14931858)) AND isnotnull(supplier_id#14931859))
(87) Project
Output [2]: [brand_context_id#14931858, supplier_id#14931859]
Input [4]: [brand_context_id#14931858, canonical_id#14931866, is_primary#14931867, supplier_id#14931859]
(88) BroadcastHashJoin
Left keys [2]: [brand_context_id#14931846, supplier_id#14931847]
Right keys [2]: [brand_context_id#14931858, supplier_id#14931859]
Join type: Inner
Join condition: None
(89) Project
Output [2]: [brand_context_id#14931846, supplier_id#14931847]
Input [4]: [brand_context_id#14931846, supplier_id#14931847, brand_context_id#14931858, supplier_id#14931859]
(90) Exchange
Input [2]: [brand_context_id#14931846, supplier_id#14931847]
Arguments: hashpartitioning(brand_context_id#14931846, supplier_id#14931847, 10), ENSURE_REQUIREMENTS, [plan_id=4291261]
(91) Sort
Input [2]: [brand_context_id#14931846, supplier_id#14931847]
Arguments: [brand_context_id#14931846 ASC NULLS FIRST, supplier_id#14931847 ASC NULLS FIRST], false, 0
(92) SortMergeJoin
Left keys [2]: [brand_context_id#14931834, supplier_id#14931835]
Right keys [2]: [brand_context_id#14931846, supplier_id#14931847]
Join type: LeftAnti
Join condition: None
(93) InMemoryTableScan
Output [7]: [brand_context_id#14931731, supplier_id#14931732, entity_name#14931733, city#14931734, state#14931735, country#14931736, threshold#14931741]
Arguments: [brand_context_id#14931731, supplier_id#14931732, entity_name#14931733, city#14931734, state#14931735, country#14931736, threshold#14931741], [isnotnull(threshold#14931741), isnotnull(brand_context_id#14931731), isnotnull(supplier_id#14931732)]
(94) InMemoryRelation
Arguments: [brand_context_id#14931731, supplier_id#14931732, entity_name#14931733, city#14931734, state#14931735, country#14931736, supplier_type#14931737, declaring_supplier_id#14931738, threshold#14931741, create_date#14931742], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@730ae8bf,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Project [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, threshold#14929533, create_date#14929534]
+- *(1) Filter isnull(canonical_id#14929531)
+- TableCacheQueryStage 0
+- InMemoryTableScan [brand_context_id#14929523, canonical_id#14929531, city#14929526, country#14929528, create_date#14929534, declaring_supplier_id#14929530, entity_name#14929525, state#14929527, supplier_id#14929524, supplier_type#14929529, threshold#14929533], [isnull(canonical_id#14929531)]
+- InMemoryRelation [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, canonical_id#14929531, is_primary#14929532, threshold#14929533, create_date#14929534], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14929523,supplier_id#14929524,entity_name#14929525,city#14929526,state#14929527,country#14929528,supplier_type#14929529,declaring_supplier_id#14929530,canonical_id#14929531,is_primary#14929532,threshold#14929533,create_date#14929534] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Project [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, threshold#14929533, create_date#14929534]
+- Filter isnull(canonical_id#14929531)
+- InMemoryTableScan [brand_context_id#14929523, canonical_id#14929531, city#14929526, country#14929528, create_date#14929534, declaring_supplier_id#14929530, entity_name#14929525, state#14929527, supplier_id#14929524, supplier_type#14929529, threshold#14929533], [isnull(canonical_id#14929531)]
+- InMemoryRelation [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, canonical_id#14929531, is_primary#14929532, threshold#14929533, create_date#14929534], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14929523,supplier_id#14929524,entity_name#14929525,city#14929526,state#14929527,country#14929528,supplier_type#14929529,declaring_supplier_id#14929530,canonical_id#14929531,is_primary#14929532,threshold#14929533,create_date#14929534] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(95) AdaptiveSparkPlan
Output [10]: [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, threshold#14929533, create_date#14929534]
Arguments: isFinalPlan=false
(96) Filter
Input [7]: [brand_context_id#14931731, supplier_id#14931732, entity_name#14931733, city#14931734, state#14931735, country#14931736, threshold#14931741]
Condition : ((isnotnull(threshold#14931741) AND isnotnull(brand_context_id#14931731)) AND isnotnull(supplier_id#14931732))
(97) BatchEvalPython
Input [7]: [brand_context_id#14931731, supplier_id#14931732, entity_name#14931733, city#14931734, state#14931735, country#14931736, threshold#14931741]
Arguments: [normalize_location(city#14931734)#14931067, normalize_location(state#14931735)#14931081, normalize_location(country#14931736)#14931096], [pythonUDF0#14933205, pythonUDF1#14933206, pythonUDF2#14933207]
(98) Filter
Input [10]: [brand_context_id#14931731, supplier_id#14931732, entity_name#14931733, city#14931734, state#14931735, country#14931736, threshold#14931741, pythonUDF0#14933205, pythonUDF1#14933206, pythonUDF2#14933207]
Condition : ((isnotnull(pythonUDF0#14933205) AND isnotnull(pythonUDF1#14933206)) AND isnotnull(pythonUDF2#14933207))
(99) Project
Output [7]: [brand_context_id#14931731, supplier_id#14931732, entity_name#14931733, city#14931734, state#14931735, country#14931736, threshold#14931741]
Input [10]: [brand_context_id#14931731, supplier_id#14931732, entity_name#14931733, city#14931734, state#14931735, country#14931736, threshold#14931741, pythonUDF0#14933205, pythonUDF1#14933206, pythonUDF2#14933207]
(100) BatchEvalPython
Input [7]: [brand_context_id#14931731, supplier_id#14931732, entity_name#14931733, city#14931734, state#14931735, country#14931736, threshold#14931741]
Arguments: [normalize_name(entity_name#14931733)#14931054, normalize_location(city#14931734)#14931067, normalize_location(state#14931735)#14931081, normalize_location(country#14931736)#14931096], [pythonUDF0#14933208, pythonUDF1#14933209, pythonUDF2#14933210, pythonUDF3#14933211]
(101) Project
Output [5]: [brand_context_id#14931731, supplier_id#14931732, threshold#14931741, pythonUDF0#14933208 AS normalized_name#14931055, concat_ws(|, coalesce(pythonUDF1#14933209, ), coalesce(pythonUDF2#14933210, ), coalesce(pythonUDF3#14933211, )) AS location_key#14931112]
Input [11]: [brand_context_id#14931731, supplier_id#14931732, entity_name#14931733, city#14931734, state#14931735, country#14931736, threshold#14931741, pythonUDF0#14933208, pythonUDF1#14933209, pythonUDF2#14933210, pythonUDF3#14933211]
(102) Exchange
Input [5]: [brand_context_id#14931731, supplier_id#14931732, threshold#14931741, normalized_name#14931055, location_key#14931112]
Arguments: hashpartitioning(brand_context_id#14931731, supplier_id#14931732, 10), ENSURE_REQUIREMENTS, [plan_id=4291270]
(103) Sort
Input [5]: [brand_context_id#14931731, supplier_id#14931732, threshold#14931741, normalized_name#14931055, location_key#14931112]
Arguments: [brand_context_id#14931731 ASC NULLS FIRST, supplier_id#14931732 ASC NULLS FIRST], false, 0
(104) InMemoryTableScan
Output [2]: [brand_context_id#14931743, supplier_id#14931744]
Arguments: [brand_context_id#14931743, supplier_id#14931744], [isnotnull(brand_context_id#14931743), isnotnull(supplier_id#14931744)]
(105) InMemoryRelation
Arguments: [brand_context_id#14931743, supplier_id#14931744, entity_name#14931745, city#14931746, state#14931747, country#14931748, supplier_type#14931749, declaring_supplier_id#14931750, threshold#14931753, create_date#14931754], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@730ae8bf,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Project [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, threshold#14929533, create_date#14929534]
+- *(1) Filter isnull(canonical_id#14929531)
+- TableCacheQueryStage 0
+- InMemoryTableScan [brand_context_id#14929523, canonical_id#14929531, city#14929526, country#14929528, create_date#14929534, declaring_supplier_id#14929530, entity_name#14929525, state#14929527, supplier_id#14929524, supplier_type#14929529, threshold#14929533], [isnull(canonical_id#14929531)]
+- InMemoryRelation [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, canonical_id#14929531, is_primary#14929532, threshold#14929533, create_date#14929534], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14929523,supplier_id#14929524,entity_name#14929525,city#14929526,state#14929527,country#14929528,supplier_type#14929529,declaring_supplier_id#14929530,canonical_id#14929531,is_primary#14929532,threshold#14929533,create_date#14929534] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Project [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, threshold#14929533, create_date#14929534]
+- Filter isnull(canonical_id#14929531)
+- InMemoryTableScan [brand_context_id#14929523, canonical_id#14929531, city#14929526, country#14929528, create_date#14929534, declaring_supplier_id#14929530, entity_name#14929525, state#14929527, supplier_id#14929524, supplier_type#14929529, threshold#14929533], [isnull(canonical_id#14929531)]
+- InMemoryRelation [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, canonical_id#14929531, is_primary#14929532, threshold#14929533, create_date#14929534], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14929523,supplier_id#14929524,entity_name#14929525,city#14929526,state#14929527,country#14929528,supplier_type#14929529,declaring_supplier_id#14929530,canonical_id#14929531,is_primary#14929532,threshold#14929533,create_date#14929534] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(106) AdaptiveSparkPlan
Output [10]: [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, threshold#14929533, create_date#14929534]
Arguments: isFinalPlan=false
(107) Filter
Input [2]: [brand_context_id#14931743, supplier_id#14931744]
Condition : (isnotnull(brand_context_id#14931743) AND isnotnull(supplier_id#14931744))
(108) BroadcastExchange
Input [2]: [brand_context_id#14931743, supplier_id#14931744]
Arguments: HashedRelationBroadcastMode(List(input[0, string, false], input[1, string, false]),false), [plan_id=4291265]
(109) InMemoryTableScan
Output [4]: [brand_context_id#14931755, canonical_id#14931763, is_primary#14931764, supplier_id#14931756]
Arguments: [brand_context_id#14931755, canonical_id#14931763, is_primary#14931764, supplier_id#14931756], [isnotnull(is_primary#14931764), is_primary#14931764, isnotnull(canonical_id#14931763), isnotnull(brand_context_id#14931755), isnotnull(supplier_id#14931756)]
(110) InMemoryRelation
Arguments: [brand_context_id#14931755, supplier_id#14931756, canonical_id#14931763, is_primary#14931764, entity_name#14931757, city#14931758, state#14931759, country#14931760, threshold#14931765, create_date#14931766], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@730ae8bf,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Filter isnotnull(canonical_id#14929531)
+- TableCacheQueryStage 0
+- InMemoryTableScan [brand_context_id#14929523, supplier_id#14929524, canonical_id#14929531, is_primary#14929532, entity_name#14929525, city#14929526, state#14929527, country#14929528, threshold#14929533, create_date#14929534], [isnotnull(canonical_id#14929531)]
+- InMemoryRelation [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, canonical_id#14929531, is_primary#14929532, threshold#14929533, create_date#14929534], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14929523,supplier_id#14929524,entity_name#14929525,city#14929526,state#14929527,country#14929528,supplier_type#14929529,declaring_supplier_id#14929530,canonical_id#14929531,is_primary#14929532,threshold#14929533,create_date#14929534] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Filter isnotnull(canonical_id#14929531)
+- InMemoryTableScan [brand_context_id#14929523, supplier_id#14929524, canonical_id#14929531, is_primary#14929532, entity_name#14929525, city#14929526, state#14929527, country#14929528, threshold#14929533, create_date#14929534], [isnotnull(canonical_id#14929531)]
+- InMemoryRelation [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, canonical_id#14929531, is_primary#14929532, threshold#14929533, create_date#14929534], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14929523,supplier_id#14929524,entity_name#14929525,city#14929526,state#14929527,country#14929528,supplier_type#14929529,declaring_supplier_id#14929530,canonical_id#14929531,is_primary#14929532,threshold#14929533,create_date#14929534] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(111) AdaptiveSparkPlan
Output [10]: [brand_context_id#14929523, supplier_id#14929524, canonical_id#14929531, is_primary#14929532, entity_name#14929525, city#14929526, state#14929527, country#14929528, threshold#14929533, create_date#14929534]
Arguments: isFinalPlan=false
(112) Filter
Input [4]: [brand_context_id#14931755, canonical_id#14931763, is_primary#14931764, supplier_id#14931756]
Condition : ((((isnotnull(is_primary#14931764) AND is_primary#14931764) AND isnotnull(canonical_id#14931763)) AND isnotnull(brand_context_id#14931755)) AND isnotnull(supplier_id#14931756))
(113) Project
Output [2]: [brand_context_id#14931755, supplier_id#14931756]
Input [4]: [brand_context_id#14931755, canonical_id#14931763, is_primary#14931764, supplier_id#14931756]
(114) BroadcastHashJoin
Left keys [2]: [brand_context_id#14931743, supplier_id#14931744]
Right keys [2]: [brand_context_id#14931755, supplier_id#14931756]
Join type: Inner
Join condition: None
(115) Project
Output [2]: [brand_context_id#14931743, supplier_id#14931744]
Input [4]: [brand_context_id#14931743, supplier_id#14931744, brand_context_id#14931755, supplier_id#14931756]
(116) Exchange
Input [2]: [brand_context_id#14931743, supplier_id#14931744]
Arguments: hashpartitioning(brand_context_id#14931743, supplier_id#14931744, 10), ENSURE_REQUIREMENTS, [plan_id=4291271]
(117) Sort
Input [2]: [brand_context_id#14931743, supplier_id#14931744]
Arguments: [brand_context_id#14931743 ASC NULLS FIRST, supplier_id#14931744 ASC NULLS FIRST], false, 0
(118) SortMergeJoin
Left keys [2]: [brand_context_id#14931731, supplier_id#14931732]
Right keys [2]: [brand_context_id#14931743, supplier_id#14931744]
Join type: LeftAnti
Join condition: None
(119) Project
Output [5]: [brand_context_id#14931731 AS u_brand_context_id#14931588, supplier_id#14931732 AS u_supplier_id#14931589, threshold#14931741 AS u_threshold#14931592, normalized_name#14931055 AS u_normalized_name#14931593, location_key#14931112 AS u_location_key#14931597]
Input [5]: [brand_context_id#14931731, supplier_id#14931732, threshold#14931741, normalized_name#14931055, location_key#14931112]
(120) BroadcastExchange
Input [5]: [u_brand_context_id#14931588, u_supplier_id#14931589, u_threshold#14931592, u_normalized_name#14931593, u_location_key#14931597]
Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[4, string, false]),false), [plan_id=4291277]
(121) InMemoryTableScan
Output [7]: [brand_context_id#14931870, supplier_id#14931871, entity_name#14931872, city#14931873, state#14931874, country#14931875, create_date#14931881]
Arguments: [brand_context_id#14931870, supplier_id#14931871, entity_name#14931872, city#14931873, state#14931874, country#14931875, create_date#14931881], [isnotnull(brand_context_id#14931870)]
(122) InMemoryRelation
Arguments: [brand_context_id#14931870, supplier_id#14931871, canonical_id#14931878, is_primary#14931879, entity_name#14931872, city#14931873, state#14931874, country#14931875, threshold#14931880, create_date#14931881], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@730ae8bf,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Filter isnotnull(canonical_id#14929531)
+- TableCacheQueryStage 0
+- InMemoryTableScan [brand_context_id#14929523, supplier_id#14929524, canonical_id#14929531, is_primary#14929532, entity_name#14929525, city#14929526, state#14929527, country#14929528, threshold#14929533, create_date#14929534], [isnotnull(canonical_id#14929531)]
+- InMemoryRelation [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, canonical_id#14929531, is_primary#14929532, threshold#14929533, create_date#14929534], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14929523,supplier_id#14929524,entity_name#14929525,city#14929526,state#14929527,country#14929528,supplier_type#14929529,declaring_supplier_id#14929530,canonical_id#14929531,is_primary#14929532,threshold#14929533,create_date#14929534] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Filter isnotnull(canonical_id#14929531)
+- InMemoryTableScan [brand_context_id#14929523, supplier_id#14929524, canonical_id#14929531, is_primary#14929532, entity_name#14929525, city#14929526, state#14929527, country#14929528, threshold#14929533, create_date#14929534], [isnotnull(canonical_id#14929531)]
+- InMemoryRelation [brand_context_id#14929523, supplier_id#14929524, entity_name#14929525, city#14929526, state#14929527, country#14929528, supplier_type#14929529, declaring_supplier_id#14929530, canonical_id#14929531, is_primary#14929532, threshold#14929533, create_date#14929534], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#14929523,supplier_id#14929524,entity_name#14929525,city#14929526,state#14929527,country#14929528,supplier_type#14929529,declaring_supplier_id#14929530,canonical_id#14929531,is_primary#14929532,threshold#14929533,create_date#14929534] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(123) AdaptiveSparkPlan
Output [10]: [brand_context_id#14929523, supplier_id#14929524, canonical_id#14929531, is_primary#14929532, entity_name#14929525, city#14929526, state#14929527, country#14929528, threshold#14929533, create_date#14929534]
Arguments: isFinalPlan=false
(124) Filter
Input [7]: [brand_context_id#14931870, supplier_id#14931871, entity_name#14931872, city#14931873, state#14931874, country#14931875, create_date#14931881]
Condition : isnotnull(brand_context_id#14931870)
(125) BatchEvalPython
Input [7]: [brand_context_id#14931870, supplier_id#14931871, entity_name#14931872, city#14931873, state#14931874, country#14931875, create_date#14931881]
Arguments: [normalize_name(entity_name#14931872)#14931128, normalize_location(city#14931873)#14931141, normalize_location(state#14931874)#14931155, normalize_location(country#14931875)#14931170], [pythonUDF0#14933212, pythonUDF1#14933213, pythonUDF2#14933214, pythonUDF3#14933215]
(126) Project
Output [5]: [brand_context_id#14931870 AS e_brand_context_id#14931610, supplier_id#14931871 AS e_supplier_id#14931611, pythonUDF0#14933212 AS e_normalized_name#14931613, concat_ws(|, coalesce(pythonUDF1#14933213, ), coalesce(pythonUDF2#14933214, ), coalesce(pythonUDF3#14933215, )) AS e_location_key#14931614, create_date#14931881 AS e_create_date#14931615]
Input [11]: [brand_context_id#14931870, supplier_id#14931871, entity_name#14931872, city#14931873, state#14931874, country#14931875, create_date#14931881, pythonUDF0#14933212, pythonUDF1#14933213, pythonUDF2#14933214, pythonUDF3#14933215]
(127) BroadcastHashJoin
Left keys [2]: [u_brand_context_id#14931588, u_location_key#14931597]
Right keys [2]: [e_brand_context_id#14931610, e_location_key#14931614]
Join type: Inner
Join condition: None
(128) Project
Output [7]: [u_brand_context_id#14931588, u_supplier_id#14931589, u_threshold#14931592, u_normalized_name#14931593, e_supplier_id#14931611, e_normalized_name#14931613, e_create_date#14931615]
Input [10]: [u_brand_context_id#14931588, u_supplier_id#14931589, u_threshold#14931592, u_normalized_name#14931593, u_location_key#14931597, e_brand_context_id#14931610, e_supplier_id#14931611, e_normalized_name#14931613, e_location_key#14931614, e_create_date#14931615]
(129) BatchEvalPython
Input [7]: [u_brand_context_id#14931588, u_supplier_id#14931589, u_threshold#14931592, u_normalized_name#14931593, e_supplier_id#14931611, e_normalized_name#14931613, e_create_date#14931615]
Arguments: [calculate_similarity(u_normalized_name#14931593, e_normalized_name#14931613)#14931668], [pythonUDF0#14933216]
(130) Filter
Input [8]: [u_brand_context_id#14931588, u_supplier_id#14931589, u_threshold#14931592, u_normalized_name#14931593, e_supplier_id#14931611, e_normalized_name#14931613, e_create_date#14931615, pythonUDF0#14933216]
Condition : (cast(pythonUDF0#14933216 as double) >= u_threshold#14931592)
(131) Project
Output [6]: [u_brand_context_id#14931588, u_supplier_id#14931589, u_normalized_name#14931593, e_supplier_id#14931611, e_normalized_name#14931613, e_create_date#14931615]
Input [8]: [u_brand_context_id#14931588, u_supplier_id#14931589, u_threshold#14931592, u_normalized_name#14931593, e_supplier_id#14931611, e_normalized_name#14931613, e_create_date#14931615, pythonUDF0#14933216]
(132) BatchEvalPython
Input [6]: [u_brand_context_id#14931588, u_supplier_id#14931589, u_normalized_name#14931593, e_supplier_id#14931611, e_normalized_name#14931613, e_create_date#14931615]
Arguments: [calculate_similarity(u_normalized_name#14931593, e_normalized_name#14931613)#14931668], [pythonUDF0#14933217]
(133) Project
Output [5]: [u_brand_context_id#14931588, u_supplier_id#14931589, e_supplier_id#14931611, e_create_date#14931615, pythonUDF0#14933217 AS similarity#14931669]
Input [7]: [u_brand_context_id#14931588, u_supplier_id#14931589, u_normalized_name#14931593, e_supplier_id#14931611, e_normalized_name#14931613, e_create_date#14931615, pythonUDF0#14933217]
(134) Sort
Input [5]: [u_brand_context_id#14931588, u_supplier_id#14931589, e_supplier_id#14931611, e_create_date#14931615, similarity#14931669]
Arguments: [u_brand_context_id#14931588 ASC NULLS FIRST, u_supplier_id#14931589 ASC NULLS FIRST, similarity#14931669 DESC NULLS LAST, e_create_date#14931615 ASC NULLS FIRST, e_supplier_id#14931611 ASC NULLS FIRST], false, 0
(135) WindowGroupLimit
Input [5]: [u_brand_context_id#14931588, u_supplier_id#14931589, e_supplier_id#14931611, e_create_date#14931615, similarity#14931669]
Arguments: [u_brand_context_id#14931588, u_supplier_id#14931589], [similarity#14931669 DESC NULLS LAST, e_create_date#14931615 ASC NULLS FIRST, e_supplier_id#14931611 ASC NULLS FIRST], row_number(), 1, Partial
(136) Exchange
Input [5]: [u_brand_context_id#14931588, u_supplier_id#14931589, e_supplier_id#14931611, e_create_date#14931615, similarity#14931669]
Arguments: hashpartitioning(u_brand_context_id#14931588, u_supplier_id#14931589, 10), ENSURE_REQUIREMENTS, [plan_id=4291289]
(137) Sort
Input [5]: [u_brand_context_id#14931588, u_supplier_id#14931589, e_supplier_id#14931611, e_create_date#14931615, similarity#14931669]
Arguments: [u_brand_context_id#14931588 ASC NULLS FIRST, u_supplier_id#14931589 ASC NULLS FIRST, similarity#14931669 DESC NULLS LAST, e_create_date#14931615 ASC NULLS FIRST, e_supplier_id#14931611 ASC NULLS FIRST], false, 0
(138) WindowGroupLimit
Input [5]: [u_brand_context_id#14931588, u_supplier_id#14931589, e_supplier_id#14931611, e_create_date#14931615, similarity#14931669]
Arguments: [u_brand_context_id#14931588, u_supplier_id#14931589], [similarity#14931669 DESC NULLS LAST, e_create_date#14931615 ASC NULLS FIRST, e_supplier_id#14931611 ASC NULLS FIRST], row_number(), 1, Final
(139) Window
Input [5]: [u_brand_context_id#14931588, u_supplier_id#14931589, e_supplier_id#14931611, e_create_date#14931615, similarity#14931669]
Arguments: [row_number() windowspecdefinition(u_brand_context_id#14931588, u_supplier_id#14931589, similarity#14931669 DESC NULLS LAST, e_create_date#14931615 ASC NULLS FIRST, e_supplier_id#14931611 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank#14931689], [u_brand_context_id#14931588, u_supplier_id#14931589], [similarity#14931669 DESC NULLS LAST, e_create_date#14931615 ASC NULLS FIRST, e_supplier_id#14931611 ASC NULLS FIRST]
(140) Filter
Input [6]: [u_brand_context_id#14931588, u_supplier_id#14931589, e_supplier_id#14931611, e_create_date#14931615, similarity#14931669, rank#14931689]
Condition : (rank#14931689 = 1)
(141) Project
Output [2]: [u_brand_context_id#14931588 AS brand_context_id#14931709, u_supplier_id#14931589 AS supplier_id#14931710]
Input [6]: [u_brand_context_id#14931588, u_supplier_id#14931589, e_supplier_id#14931611, e_create_date#14931615, similarity#14931669, rank#14931689]
(142) BroadcastExchange
Input [2]: [brand_context_id#14931709, supplier_id#14931710]
Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true]),false), [plan_id=4291296]
(143) BroadcastHashJoin
Left keys [2]: [brand_context_id#14931834, supplier_id#14931835]
Right keys [2]: [brand_context_id#14931709, supplier_id#14931710]
Join type: LeftAnti
Join condition: None
(144) Project
Output [10]: [brand_context_id#14931834, supplier_id#14931835, declaring_supplier_id#14931841, supplier_type#14931840, supplier_id#14931835 AS canonical_id#14931782, 1 AS is_primary#14931783, threshold#14931844 AS configured_threshold#14931784, 0.0 AS match_percentage#14931785, 0.0 AS confidence_score#14931786, new_canonical AS match_reason#14931787]
Input [5]: [brand_context_id#14931834, supplier_id#14931835, supplier_type#14931840, declaring_supplier_id#14931841, threshold#14931844]
(145) Union
(146) AdaptiveSparkPlan
Output [10]: [brand_context_id#14929523, supplier_id#14929524, declaring_supplier_id#14929530, supplier_type#14929529, canonical_id#14931480, is_primary#14931544, configured_threshold#14931545, match_percentage#14931546, confidence_score#14931547, match_reason#14931548]
Arguments: isFinalPlan=false
(147) TableCacheQueryStage
Output [10]: [brand_context_id#14929523, canonical_id#14931480, confidence_score#14931547, configured_threshold#14931545, declaring_supplier_id#14929530, is_primary#14931544, match_percentage#14931546, match_reason#14931548, supplier_id#14929524, supplier_type#14929529]
Arguments: 0
(148) Project [codegen id : 1]
Output [12]: [brand_context_id#14929523, supplier_id#14929524, declaring_supplier_id#14929530, supplier_type#14929529, canonical_id#14931480, is_primary#14931544, configured_threshold#14931545, match_percentage#14931546, confidence_score#14931547, match_reason#14931548, 2026-10-19 03:23:09.111953 AS created_at#14935699, 2026-10-19 03:23:09.111953 AS updated_at#14935711]
Input [10]: [brand_context_id#14929523, canonical_id#14931480, confidence_score#14931547, configured_threshold#14931545, declaring_supplier_id#14929530, is_primary#14931544, match_percentage#14931546, match_reason#14931548, supplier_id#14929524, supplier_type#14929529]
(149) DeserializeToObject
Input [12]: [brand_context_id#14929523, supplier_id#14929524, declaring_supplier_id#14929530, supplier_type#14929529, canonical_id#14931480, is_primary#14931544, configured_threshold#14931545, match_percentage#14931546, confidence_score#14931547, match_reason#14931548, created_at#14935699, updated_at#14935711]
Arguments: createexternalrow(brand_context_id#14929523.toString, supplier_id#14929524.toString, declaring_supplier_id#14929530.toString, supplier_type#14929529.toString, canonical_id#14931480.toString, staticinvoke(class java.lang.Integer, ObjectType(class java.lang.Integer), valueOf, is_primary#14931544, true, false, true), staticinvoke(class java.lang.Double, ObjectType(class java.lang.Double), valueOf, configured_threshold#14931545, true, false, true), staticinvoke(class java.lang.Double, ObjectType(class java.lang.Double), valueOf, match_percentage#14931546, true, false, true), staticinvoke(class java.lang.Double, ObjectType(class java.lang.Double), valueOf, confidence_score#14931547, true, false, true), match_reason#14931548.toString, staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Timestamp), toJavaTimestamp, created_at#14935699, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Timestamp), toJavaTimestamp, updated_at#14935711, true, false, true), StructField(brand_context_id,StringType,true), StructField(supplier_id,StringType,true), StructField(declaring_supplier_id,StringType,true), StructField(supplier_type,StringType,true), StructField(canonical_id,StringType,true), StructField(is_primary,IntegerType,false), StructField(configured_threshold,DoubleType,true), StructField(match_percentage,DoubleType,true), StructField(confidence_score,DoubleType,true), StructField(match_reason,StringType,false), StructField(created_at,TimestampType,false), StructField(updated_at,TimestampType,false)), obj#14936015: org.apache.spark.sql.Row
(150) Project
Output [12]: [brand_context_id#14929523, supplier_id#14929524, declaring_supplier_id#14929530, supplier_type#14929529, canonical_id#14931480, is_primary#14931544, configured_threshold#14931545, match_percentage#14931546, confidence_score#14931547, match_reason#14931548, 2026-10-19 03:23:09.111953 AS created_at#14935699, 2026-10-19 03:23:09.111953 AS updated_at#14935711]
Input [10]: [brand_context_id#14929523, canonical_id#14931480, confidence_score#14931547, configured_threshold#14931545, declaring_supplier_id#14929530, is_primary#14931544, match_percentage#14931546, match_reason#14931548, supplier_id#14929524, supplier_type#14929529]
(151) DeserializeToObject
Input [12]: [brand_context_id#14929523, supplier_id#14929524, declaring_supplier_id#14929530, supplier_type#14929529, canonical_id#14931480, is_primary#14931544, configured_threshold#14931545, match_percentage#14931546, confidence_score#14931547, match_reason#14931548, created_at#14935699, updated_at#14935711]
Arguments: createexternalrow(brand_context_id#14929523.toString, supplier_id#14929524.toString, declaring_supplier_id#14929530.toString, supplier_type#14929529.toString, canonical_id#14931480.toString, staticinvoke(class java.lang.Integer, ObjectType(class java.lang.Integer), valueOf, is_primary#14931544, true, false, true), staticinvoke(class java.lang.Double, ObjectType(class java.lang.Double), valueOf, configured_threshold#14931545, true, false, true), staticinvoke(class java.lang.Double, ObjectType(class java.lang.Double), valueOf, match_percentage#14931546, true, false, true), staticinvoke(class java.lang.Double, ObjectType(class java.lang.Double), valueOf, confidence_score#14931547, true, false, true), match_reason#14931548.toString, staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Timestamp), toJavaTimestamp, created_at#14935699, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Timestamp), toJavaTimestamp, updated_at#14935711, true, false, true), StructField(brand_context_id,StringType,true), StructField(supplier_id,StringType,true), StructField(declaring_supplier_id,StringType,true), StructField(supplier_type,StringType,true), StructField(canonical_id,StringType,true), StructField(is_primary,IntegerType,false), StructField(configured_threshold,DoubleType,true), StructField(match_percentage,DoubleType,true), StructField(confidence_score,DoubleType,true), StructField(match_reason,StringType,false), StructField(created_at,TimestampType,false), StructField(updated_at,TimestampType,false)), obj#14936015: org.apache.spark.sql.Row
(152) AdaptiveSparkPlan
Output [1]: [obj#14936015]
Arguments: isFinalPlan=true