== Physical Plan ==
AdaptiveSparkPlan (155)
+- == Final Plan ==
* HashAggregate (151)
+- AQEShuffleRead (150)
+- ShuffleQueryStage (149), Statistics(sizeInBytes=160.0 B, rowCount=4)
+- Exchange (148)
+- * HashAggregate (147)
+- TableCacheQueryStage (146), Statistics(sizeInBytes=902.0 B, rowCount=5)
+- InMemoryTableScan (1)
+- InMemoryRelation (2)
+- AdaptiveSparkPlan (145)
+- Union (144)
:- Project (23)
: +- BroadcastHashJoin Inner BuildLeft (22)
: :- BroadcastExchange (12)
: : +- Filter (11)
: : +- InMemoryTableScan (3)
: : +- InMemoryRelation (4)
: : +- AdaptiveSparkPlan (10)
: : +- Project (9)
: : +- Filter (8)
: : +- InMemoryTableScan (5)
: : +- InMemoryRelation (6)
: : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: +- Project (21)
: +- Filter (20)
: +- InMemoryTableScan (13)
: +- InMemoryRelation (14)
: +- AdaptiveSparkPlan (19)
: +- Filter (18)
: +- InMemoryTableScan (15)
: +- InMemoryRelation (16)
: +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (17)
:- Project (72)
: +- Filter (71)
: +- Window (70)
: +- WindowGroupLimit (69)
: +- Sort (68)
: +- Exchange (67)
: +- WindowGroupLimit (66)
: +- Sort (65)
: +- Project (64)
: +- BatchEvalPython (63)
: +- Project (62)
: +- Filter (61)
: +- BatchEvalPython (60)
: +- Project (59)
: +- BroadcastHashJoin Inner BuildLeft (58)
: :- BroadcastExchange (51)
: : +- Project (50)
: : +- SortMergeJoin LeftAnti (49)
: : :- Sort (34)
: : : +- Exchange (33)
: : : +- Project (32)
: : : +- BatchEvalPython (31)
: : : +- Project (30)
: : : +- Filter (29)
: : : +- BatchEvalPython (28)
: : : +- Filter (27)
: : : +- InMemoryTableScan (24)
: : : +- InMemoryRelation (25)
: : : +- AdaptiveSparkPlan (26)
: : : +- Project (9)
: : : +- Filter (8)
: : : +- InMemoryTableScan (5)
: : : +- InMemoryRelation (6)
: : : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: : +- Sort (48)
: : +- Exchange (47)
: : +- Project (46)
: : +- BroadcastHashJoin Inner BuildLeft (45)
: : :- BroadcastExchange (39)
: : : +- Filter (38)
: : : +- InMemoryTableScan (35)
: : : +- InMemoryRelation (36)
: : : +- AdaptiveSparkPlan (37)
: : : +- Project (9)
: : : +- Filter (8)
: : : +- InMemoryTableScan (5)
: : : +- InMemoryRelation (6)
: : : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: : +- Project (44)
: : +- Filter (43)
: : +- InMemoryTableScan (40)
: : +- InMemoryRelation (41)
: : +- AdaptiveSparkPlan (42)
: : +- Filter (18)
: : +- InMemoryTableScan (15)
: : +- InMemoryRelation (16)
: : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (17)
: +- Project (57)
: +- BatchEvalPython (56)
: +- Filter (55)
: +- InMemoryTableScan (52)
: +- InMemoryRelation (53)
: +- AdaptiveSparkPlan (54)
: +- Filter (18)
: +- InMemoryTableScan (15)
: +- InMemoryRelation (16)
: +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (17)
+- Project (143)
+- SortMergeJoin LeftAnti (142)
:- SortMergeJoin LeftAnti (92)
: :- Sort (77)
: : +- Exchange (76)
: : +- InMemoryTableScan (73)
: : +- InMemoryRelation (74)
: : +- AdaptiveSparkPlan (75)
: : +- Project (9)
: : +- Filter (8)
: : +- InMemoryTableScan (5)
: : +- InMemoryRelation (6)
: : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: +- Sort (91)
: +- Exchange (90)
: +- Project (89)
: +- BroadcastHashJoin Inner BuildLeft (88)
: :- BroadcastExchange (82)
: : +- Filter (81)
: : +- InMemoryTableScan (78)
: : +- InMemoryRelation (79)
: : +- AdaptiveSparkPlan (80)
: : +- Project (9)
: : +- Filter (8)
: : +- InMemoryTableScan (5)
: : +- InMemoryRelation (6)
: : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: +- Project (87)
: +- Filter (86)
: +- InMemoryTableScan (83)
: +- InMemoryRelation (84)
: +- AdaptiveSparkPlan (85)
: +- Filter (18)
: +- InMemoryTableScan (15)
: +- InMemoryRelation (16)
: +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (17)
+- Project (141)
+- Filter (140)
+- Window (139)
+- WindowGroupLimit (138)
+- Sort (137)
+- Exchange (136)
+- WindowGroupLimit (135)
+- Sort (134)
+- Project (133)
+- BatchEvalPython (132)
+- Project (131)
+- Filter (130)
+- BatchEvalPython (129)
+- Project (128)
+- BroadcastHashJoin Inner BuildLeft (127)
:- BroadcastExchange (120)
: +- Project (119)
: +- SortMergeJoin LeftAnti (118)
: :- Sort (103)
: : +- Exchange (102)
: : +- Project (101)
: : +- BatchEvalPython (100)
: : +- Project (99)
: : +- Filter (98)
: : +- BatchEvalPython (97)
: : +- Filter (96)
: : +- InMemoryTableScan (93)
: : +- InMemoryRelation (94)
: : +- AdaptiveSparkPlan (95)
: : +- Project (9)
: : +- Filter (8)
: : +- InMemoryTableScan (5)
: : +- InMemoryRelation (6)
: : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: +- Sort (117)
: +- Exchange (116)
: +- Project (115)
: +- BroadcastHashJoin Inner BuildLeft (114)
: :- BroadcastExchange (108)
: : +- Filter (107)
: : +- InMemoryTableScan (104)
: : +- InMemoryRelation (105)
: : +- AdaptiveSparkPlan (106)
: : +- Project (9)
: : +- Filter (8)
: : +- InMemoryTableScan (5)
: : +- InMemoryRelation (6)
: : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: +- Project (113)
: +- Filter (112)
: +- InMemoryTableScan (109)
: +- InMemoryRelation (110)
: +- AdaptiveSparkPlan (111)
: +- Filter (18)
: +- InMemoryTableScan (15)
: +- InMemoryRelation (16)
: +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (17)
+- Project (126)
+- BatchEvalPython (125)
+- Filter (124)
+- InMemoryTableScan (121)
+- InMemoryRelation (122)
+- AdaptiveSparkPlan (123)
+- Filter (18)
+- InMemoryTableScan (15)
+- InMemoryRelation (16)
+- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (17)
+- == Initial Plan ==
HashAggregate (154)
+- Exchange (153)
+- HashAggregate (152)
+- InMemoryTableScan (1)
+- InMemoryRelation (2)
+- AdaptiveSparkPlan (145)
+- Union (144)
:- Project (23)
: +- BroadcastHashJoin Inner BuildLeft (22)
: :- BroadcastExchange (12)
: : +- Filter (11)
: : +- InMemoryTableScan (3)
: : +- InMemoryRelation (4)
: : +- AdaptiveSparkPlan (10)
: : +- Project (9)
: : +- Filter (8)
: : +- InMemoryTableScan (5)
: : +- InMemoryRelation (6)
: : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: +- Project (21)
: +- Filter (20)
: +- InMemoryTableScan (13)
: +- InMemoryRelation (14)
: +- AdaptiveSparkPlan (19)
: +- Filter (18)
: +- InMemoryTableScan (15)
: +- InMemoryRelation (16)
: +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (17)
:- Project (72)
: +- Filter (71)
: +- Window (70)
: +- WindowGroupLimit (69)
: +- Sort (68)
: +- Exchange (67)
: +- WindowGroupLimit (66)
: +- Sort (65)
: +- Project (64)
: +- BatchEvalPython (63)
: +- Project (62)
: +- Filter (61)
: +- BatchEvalPython (60)
: +- Project (59)
: +- BroadcastHashJoin Inner BuildLeft (58)
: :- BroadcastExchange (51)
: : +- Project (50)
: : +- SortMergeJoin LeftAnti (49)
: : :- Sort (34)
: : : +- Exchange (33)
: : : +- Project (32)
: : : +- BatchEvalPython (31)
: : : +- Project (30)
: : : +- Filter (29)
: : : +- BatchEvalPython (28)
: : : +- Filter (27)
: : : +- InMemoryTableScan (24)
: : : +- InMemoryRelation (25)
: : : +- AdaptiveSparkPlan (26)
: : : +- Project (9)
: : : +- Filter (8)
: : : +- InMemoryTableScan (5)
: : : +- InMemoryRelation (6)
: : : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: : +- Sort (48)
: : +- Exchange (47)
: : +- Project (46)
: : +- BroadcastHashJoin Inner BuildLeft (45)
: : :- BroadcastExchange (39)
: : : +- Filter (38)
: : : +- InMemoryTableScan (35)
: : : +- InMemoryRelation (36)
: : : +- AdaptiveSparkPlan (37)
: : : +- Project (9)
: : : +- Filter (8)
: : : +- InMemoryTableScan (5)
: : : +- InMemoryRelation (6)
: : : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: : +- Project (44)
: : +- Filter (43)
: : +- InMemoryTableScan (40)
: : +- InMemoryRelation (41)
: : +- AdaptiveSparkPlan (42)
: : +- Filter (18)
: : +- InMemoryTableScan (15)
: : +- InMemoryRelation (16)
: : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (17)
: +- Project (57)
: +- BatchEvalPython (56)
: +- Filter (55)
: +- InMemoryTableScan (52)
: +- InMemoryRelation (53)
: +- AdaptiveSparkPlan (54)
: +- Filter (18)
: +- InMemoryTableScan (15)
: +- InMemoryRelation (16)
: +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (17)
+- Project (143)
+- SortMergeJoin LeftAnti (142)
:- SortMergeJoin LeftAnti (92)
: :- Sort (77)
: : +- Exchange (76)
: : +- InMemoryTableScan (73)
: : +- InMemoryRelation (74)
: : +- AdaptiveSparkPlan (75)
: : +- Project (9)
: : +- Filter (8)
: : +- InMemoryTableScan (5)
: : +- InMemoryRelation (6)
: : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: +- Sort (91)
: +- Exchange (90)
: +- Project (89)
: +- BroadcastHashJoin Inner BuildLeft (88)
: :- BroadcastExchange (82)
: : +- Filter (81)
: : +- InMemoryTableScan (78)
: : +- InMemoryRelation (79)
: : +- AdaptiveSparkPlan (80)
: : +- Project (9)
: : +- Filter (8)
: : +- InMemoryTableScan (5)
: : +- InMemoryRelation (6)
: : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: +- Project (87)
: +- Filter (86)
: +- InMemoryTableScan (83)
: +- InMemoryRelation (84)
: +- AdaptiveSparkPlan (85)
: +- Filter (18)
: +- InMemoryTableScan (15)
: +- InMemoryRelation (16)
: +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (17)
+- Project (141)
+- Filter (140)
+- Window (139)
+- WindowGroupLimit (138)
+- Sort (137)
+- Exchange (136)
+- WindowGroupLimit (135)
+- Sort (134)
+- Project (133)
+- BatchEvalPython (132)
+- Project (131)
+- Filter (130)
+- BatchEvalPython (129)
+- Project (128)
+- BroadcastHashJoin Inner BuildLeft (127)
:- BroadcastExchange (120)
: +- Project (119)
: +- SortMergeJoin LeftAnti (118)
: :- Sort (103)
: : +- Exchange (102)
: : +- Project (101)
: : +- BatchEvalPython (100)
: : +- Project (99)
: : +- Filter (98)
: : +- BatchEvalPython (97)
: : +- Filter (96)
: : +- InMemoryTableScan (93)
: : +- InMemoryRelation (94)
: : +- AdaptiveSparkPlan (95)
: : +- Project (9)
: : +- Filter (8)
: : +- InMemoryTableScan (5)
: : +- InMemoryRelation (6)
: : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: +- Sort (117)
: +- Exchange (116)
: +- Project (115)
: +- BroadcastHashJoin Inner BuildLeft (114)
: :- BroadcastExchange (108)
: : +- Filter (107)
: : +- InMemoryTableScan (104)
: : +- InMemoryRelation (105)
: : +- AdaptiveSparkPlan (106)
: : +- Project (9)
: : +- Filter (8)
: : +- InMemoryTableScan (5)
: : +- InMemoryRelation (6)
: : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: +- Project (113)
: +- Filter (112)
: +- InMemoryTableScan (109)
: +- InMemoryRelation (110)
: +- AdaptiveSparkPlan (111)
: +- Filter (18)
: +- InMemoryTableScan (15)
: +- InMemoryRelation (16)
: +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (17)
+- Project (126)
+- BatchEvalPython (125)
+- Filter (124)
+- InMemoryTableScan (121)
+- InMemoryRelation (122)
+- AdaptiveSparkPlan (123)
+- Filter (18)
+- InMemoryTableScan (15)
+- InMemoryRelation (16)
+- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (17)
(1) InMemoryTableScan
Output [1]: [match_reason#11438416]
Arguments: [match_reason#11438416]
(2) InMemoryRelation
Arguments: [brand_context_id#11412129, supplier_id#11412130, declaring_supplier_id#11412136, supplier_type#11412135, canonical_id#11438340, is_primary#11438410, configured_threshold#11438411, match_percentage#11438412, confidence_score#11438415, match_reason#11438416], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@7795a5ad,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
Union
:- *(26) Project [brand_context_id#11412129, supplier_id#11412130, declaring_supplier_id#11412136, supplier_type#11412135, canonical_id#11438340, 0 AS is_primary#11438410, threshold#11412139 AS configured_threshold#11438411, 100.0 AS match_percentage#11438412, 1.0 AS confidence_score#11438415, exact_id_match AS match_reason#11438416]
: +- *(26) BroadcastHashJoin [brand_context_id#11412129, supplier_id#11412130], [brand_context_id#11438332, supplier_id#11438333], Inner, BuildLeft, false
: :- BroadcastQueryStage 15
: : +- BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, false], input[1, string, false]),false), [plan_id=581300]
: : +- *(5) Filter (isnotnull(brand_context_id#11412129) AND isnotnull(supplier_id#11412130))
: : +- TableCacheQueryStage 0
: : +- InMemoryTableScan [brand_context_id#11412129, supplier_id#11412130, supplier_type#11412135, declaring_supplier_id#11412136, threshold#11412139], [isnotnull(brand_context_id#11412129), isnotnull(supplier_id#11412130)]
: : +- InMemoryRelation [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, threshold#11412139, create_date#11412140], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- AdaptiveSparkPlan isFinalPlan=false
: : +- Project [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, threshold#11412139, create_date#11412140]
: : +- Filter isnull(canonical_id#11412137)
: : +- InMemoryTableScan [brand_context_id#11412129, canonical_id#11412137, city#11412132, country#11412134, create_date#11412140, declaring_supplier_id#11412136, entity_name#11412131, state#11412133, supplier_id#11412130, supplier_type#11412135, threshold#11412139], [isnull(canonical_id#11412137)]
: : +- InMemoryRelation [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, canonical_id#11412137, is_primary#11412138, threshold#11412139, create_date#11412140], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#11412129,supplier_id#11412130,entity_name#11412131,city#11412132,state#11412133,country#11412134,supplier_type#11412135,declaring_supplier_id#11412136,canonical_id#11412137,is_primary#11412138,threshold#11412139,create_date#11412140] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
: +- *(26) Project [brand_context_id#11438332, supplier_id#11438333, canonical_id#11438340]
: +- *(26) Filter ((((isnotnull(is_primary#11438341) AND is_primary#11438341) AND isnotnull(canonical_id#11438340)) AND isnotnull(brand_context_id#11438332)) AND isnotnull(supplier_id#11438333))
: +- TableCacheQueryStage 1
: +- InMemoryTableScan [brand_context_id#11438332, canonical_id#11438340, is_primary#11438341, supplier_id#11438333], [isnotnull(is_primary#11438341), is_primary#11438341, isnotnull(canonical_id#11438340), isnotnull(brand_context_id#11438332), isnotnull(supplier_id#11438333)]
: +- InMemoryRelation [brand_context_id#11438332, supplier_id#11438333, canonical_id#11438340, is_primary#11438341, entity_name#11438334, city#11438335, state#11438336, country#11438337, threshold#11438342, create_date#11438343], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- AdaptiveSparkPlan isFinalPlan=false
: +- Filter isnotnull(canonical_id#11412137)
: +- InMemoryTableScan [brand_context_id#11412129, supplier_id#11412130, canonical_id#11412137, is_primary#11412138, entity_name#11412131, city#11412132, state#11412133, country#11412134, threshold#11412139, create_date#11412140], [isnotnull(canonical_id#11412137)]
: +- InMemoryRelation [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, canonical_id#11412137, is_primary#11412138, threshold#11412139, create_date#11412140], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#11412129,supplier_id#11412130,entity_name#11412131,city#11412132,state#11412133,country#11412134,supplier_type#11412135,declaring_supplier_id#11412136,canonical_id#11412137,is_primary#11412138,threshold#11412139,create_date#11412140] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
:- *(28) Project [u_brand_context_id#11438504 AS brand_context_id#11440593, u_supplier_id#11438570 AS supplier_id#11440594, u_declaring_supplier_id#11438676 AS declaring_supplier_id#11440595, u_supplier_type#11438677 AS supplier_type#11440596, e_canonical_id#11438805 AS canonical_id#11440617, 0 AS is_primary#11440618, u_threshold#11438680 AS configured_threshold#11440619, cast((similarity#11440052 * 100.0) as double) AS match_percentage#11440976, cast(similarity#11440052 as double) AS confidence_score#11440977, matched_existing AS match_reason#11440622]
: +- *(28) Filter (rank#11440316 = 1)
: +- Window [row_number() windowspecdefinition(u_brand_context_id#11438504, u_supplier_id#11438570, similarity#11440052 DESC NULLS LAST, e_create_date#11438926 ASC NULLS FIRST, e_supplier_id#11438804 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank#11440316], [u_brand_context_id#11438504, u_supplier_id#11438570], [similarity#11440052 DESC NULLS LAST, e_create_date#11438926 ASC NULLS FIRST, e_supplier_id#11438804 ASC NULLS FIRST]
: +- WindowGroupLimit [u_brand_context_id#11438504, u_supplier_id#11438570], [similarity#11440052 DESC NULLS LAST, e_create_date#11438926 ASC NULLS FIRST, e_supplier_id#11438804 ASC NULLS FIRST], row_number(), 1, Final
: +- *(27) Sort [u_brand_context_id#11438504 ASC NULLS FIRST, u_supplier_id#11438570 ASC NULLS FIRST, similarity#11440052 DESC NULLS LAST, e_create_date#11438926 ASC NULLS FIRST, e_supplier_id#11438804 ASC NULLS FIRST], false, 0
: +- ShuffleQueryStage 25
: +- Exchange hashpartitioning(u_brand_context_id#11438504, u_supplier_id#11438570, 10), ENSURE_REQUIREMENTS, [plan_id=584093]
: +- WindowGroupLimit [u_brand_context_id#11438504, u_supplier_id#11438570], [similarity#11440052 DESC NULLS LAST, e_create_date#11438926 ASC NULLS FIRST, e_supplier_id#11438804 ASC NULLS FIRST], row_number(), 1, Partial
: +- *(19) Sort [u_brand_context_id#11438504 ASC NULLS FIRST, u_supplier_id#11438570 ASC NULLS FIRST, similarity#11440052 DESC NULLS LAST, e_create_date#11438926 ASC NULLS FIRST, e_supplier_id#11438804 ASC NULLS FIRST], false, 0
: +- *(19) Project [u_brand_context_id#11438504, u_supplier_id#11438570, u_declaring_supplier_id#11438676, u_supplier_type#11438677, u_threshold#11438680, e_supplier_id#11438804, e_canonical_id#11438805, e_create_date#11438926, pythonUDF0#11442429 AS similarity#11440052]
: +- BatchEvalPython [calculate_similarity(u_normalized_name#11438681, e_normalized_name#11438850)#11440051], [pythonUDF0#11442429]
: +- *(18) Project [u_brand_context_id#11438504, u_supplier_id#11438570, u_declaring_supplier_id#11438676, u_supplier_type#11438677, u_threshold#11438680, u_normalized_name#11438681, e_supplier_id#11438804, e_canonical_id#11438805, e_normalized_name#11438850, e_create_date#11438926]
: +- *(18) Filter (cast(pythonUDF0#11442428 as double) >= u_threshold#11438680)
: +- BatchEvalPython [calculate_similarity(u_normalized_name#11438681, e_normalized_name#11438850)#11440051], [pythonUDF0#11442428]
: +- *(17) Project [u_brand_context_id#11438504, u_supplier_id#11438570, u_declaring_supplier_id#11438676, u_supplier_type#11438677, u_threshold#11438680, u_normalized_name#11438681, e_supplier_id#11438804, e_canonical_id#11438805, e_normalized_name#11438850, e_create_date#11438926]
: +- *(17) BroadcastHashJoin [u_brand_context_id#11438504, u_location_key#11438790], [e_brand_context_id#11438803, e_location_key#11438896], Inner, BuildLeft, false
: :- BroadcastQueryStage 23
: : +- BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, true], input[6, string, false]),false), [plan_id=583249]
: : +- *(14) Project [brand_context_id#11440771 AS u_brand_context_id#11438504, supplier_id#11440772 AS u_supplier_id#11438570, declaring_supplier_id#11440778 AS u_declaring_supplier_id#11438676, supplier_type#11440777 AS u_supplier_type#11438677, threshold#11440781 AS u_threshold#11438680, normalized_name#11437594 AS u_normalized_name#11438681, location_key#11437651 AS u_location_key#11438790]
: : +- AQEShuffleRead coalesced
: : +- ShuffleQueryStage 13
: : +- Exchange hashpartitioning(brand_context_id#11440771, supplier_id#11440772, 10), ENSURE_REQUIREMENTS, [plan_id=580924]
: : +- *(3) Project [brand_context_id#11440771, supplier_id#11440772, supplier_type#11440777, declaring_supplier_id#11440778, threshold#11440781, pythonUDF0#11442420 AS normalized_name#11437594, concat_ws(|, coalesce(pythonUDF1#11442421, ), coalesce(pythonUDF2#11442422, ), coalesce(pythonUDF3#11442423, )) AS location_key#11437651]
: : +- BatchEvalPython [normalize_name(entity_name#11440773)#11437593, normalize_location(city#11440774)#11437606, normalize_location(state#11440775)#11437620, normalize_location(country#11440776)#11437635], [pythonUDF0#11442420, pythonUDF1#11442421, pythonUDF2#11442422, pythonUDF3#11442423]
: : +- *(2) Project [brand_context_id#11440771, supplier_id#11440772, entity_name#11440773, city#11440774, state#11440775, country#11440776, supplier_type#11440777, declaring_supplier_id#11440778, threshold#11440781]
: : +- *(2) Filter ((isnotnull(pythonUDF0#11442417) AND isnotnull(pythonUDF1#11442418)) AND isnotnull(pythonUDF2#11442419))
: : +- BatchEvalPython [normalize_location(city#11440774)#11437606, normalize_location(state#11440775)#11437620, normalize_location(country#11440776)#11437635], [pythonUDF0#11442417, pythonUDF1#11442418, pythonUDF2#11442419]
: : +- *(1) Filter (isnotnull(threshold#11440781) AND isnotnull(brand_context_id#11440771))
: : +- TableCacheQueryStage 2
: : +- InMemoryTableScan [brand_context_id#11440771, supplier_id#11440772, entity_name#11440773, city#11440774, state#11440775, country#11440776, supplier_type#11440777, declaring_supplier_id#11440778, threshold#11440781], [isnotnull(threshold#11440781), isnotnull(brand_context_id#11440771)]
: : +- InMemoryRelation [brand_context_id#11440771, supplier_id#11440772, entity_name#11440773, city#11440774, state#11440775, country#11440776, supplier_type#11440777, declaring_supplier_id#11440778, threshold#11440781, create_date#11440782], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- AdaptiveSparkPlan isFinalPlan=false
: : +- Project [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, threshold#11412139, create_date#11412140]
: : +- Filter isnull(canonical_id#11412137)
: : +- InMemoryTableScan [brand_context_id#11412129, canonical_id#11412137, city#11412132, country#11412134, create_date#11412140, declaring_supplier_id#11412136, entity_name#11412131, state#11412133, supplier_id#11412130, supplier_type#11412135, threshold#11412139], [isnull(canonical_id#11412137)]
: : +- InMemoryRelation [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, canonical_id#11412137, is_primary#11412138, threshold#11412139, create_date#11412140], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#11412129,supplier_id#11412130,entity_name#11412131,city#11412132,state#11412133,country#11412134,supplier_type#11412135,declaring_supplier_id#11412136,canonical_id#11412137,is_primary#11412138,threshold#11412139,create_date#11412140] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
: +- *(17) Project [brand_context_id#11439375 AS e_brand_context_id#11438803, supplier_id#11439376 AS e_supplier_id#11438804, canonical_id#11439383 AS e_canonical_id#11438805, pythonUDF0#11442424 AS e_normalized_name#11438850, concat_ws(|, coalesce(pythonUDF1#11442425, ), coalesce(pythonUDF2#11442426, ), coalesce(pythonUDF3#11442427, )) AS e_location_key#11438896, create_date#11439386 AS e_create_date#11438926]
: +- BatchEvalPython [normalize_name(entity_name#11439377)#11437672, normalize_location(city#11439378)#11437732, normalize_location(state#11439379)#11437750, normalize_location(country#11439380)#11437769], [pythonUDF0#11442424, pythonUDF1#11442425, pythonUDF2#11442426, pythonUDF3#11442427]
: +- *(16) Filter isnotnull(brand_context_id#11439375)
: +- TableCacheQueryStage 5
: +- InMemoryTableScan [brand_context_id#11439375, supplier_id#11439376, canonical_id#11439383, entity_name#11439377, city#11439378, state#11439379, country#11439380, create_date#11439386], [isnotnull(brand_context_id#11439375)]
: +- InMemoryRelation [brand_context_id#11439375, supplier_id#11439376, canonical_id#11439383, is_primary#11439384, entity_name#11439377, city#11439378, state#11439379, country#11439380, threshold#11439385, create_date#11439386], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- AdaptiveSparkPlan isFinalPlan=false
: +- Filter isnotnull(canonical_id#11412137)
: +- InMemoryTableScan [brand_context_id#11412129, supplier_id#11412130, canonical_id#11412137, is_primary#11412138, entity_name#11412131, city#11412132, state#11412133, country#11412134, threshold#11412139, create_date#11412140], [isnotnull(canonical_id#11412137)]
: +- InMemoryRelation [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, canonical_id#11412137, is_primary#11412138, threshold#11412139, create_date#11412140], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#11412129,supplier_id#11412130,entity_name#11412131,city#11412132,state#11412133,country#11412134,supplier_type#11412135,declaring_supplier_id#11412136,canonical_id#11412137,is_primary#11412138,threshold#11412139,create_date#11412140] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- *(29) Project [brand_context_id#11441045, supplier_id#11441046, declaring_supplier_id#11441052, supplier_type#11441051, supplier_id#11441046 AS canonical_id#11440739, 1 AS is_primary#11440756, threshold#11441055 AS configured_threshold#11440757, 0.0 AS match_percentage#11440758, 0.0 AS confidence_score#11440759, new_canonical AS match_reason#11440760]
+- *(29) BroadcastHashJoin [brand_context_id#11441045, supplier_id#11441046], [brand_context_id#11440593, supplier_id#11440594], LeftAnti, BuildRight, false
:- ShuffleQueryStage 16
: +- Exchange hashpartitioning(brand_context_id#11441045, supplier_id#11441046, 10), ENSURE_REQUIREMENTS, [plan_id=580447]
: +- TableCacheQueryStage 6
: +- InMemoryTableScan [brand_context_id#11441045, supplier_id#11441046, supplier_type#11441051, declaring_supplier_id#11441052, threshold#11441055]
: +- InMemoryRelation [brand_context_id#11441045, supplier_id#11441046, entity_name#11441047, city#11441048, state#11441049, country#11441050, supplier_type#11441051, declaring_supplier_id#11441052, threshold#11441055, create_date#11441056], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- AdaptiveSparkPlan isFinalPlan=false
: +- Project [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, threshold#11412139, create_date#11412140]
: +- Filter isnull(canonical_id#11412137)
: +- InMemoryTableScan [brand_context_id#11412129, canonical_id#11412137, city#11412132, country#11412134, create_date#11412140, declaring_supplier_id#11412136, entity_name#11412131, state#11412133, supplier_id#11412130, supplier_type#11412135, threshold#11412139], [isnull(canonical_id#11412137)]
: +- InMemoryRelation [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, canonical_id#11412137, is_primary#11412138, threshold#11412139, create_date#11412140], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#11412129,supplier_id#11412130,entity_name#11412131,city#11412132,state#11412133,country#11412134,supplier_type#11412135,declaring_supplier_id#11412136,canonical_id#11412137,is_primary#11412138,threshold#11412139,create_date#11412140] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- BroadcastQueryStage 27
+- BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true]),false), [plan_id=586889]
+- *(25) Project [u_brand_context_id#11438504 AS brand_context_id#11440593, u_supplier_id#11438570 AS supplier_id#11440594]
+- *(25) Filter (rank#11440316 = 1)
+- Window [row_number() windowspecdefinition(u_brand_context_id#11438504, u_supplier_id#11438570, similarity#11440052 DESC NULLS LAST, e_create_date#11438926 ASC NULLS FIRST, e_supplier_id#11438804 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank#11440316], [u_brand_context_id#11438504, u_supplier_id#11438570], [similarity#11440052 DESC NULLS LAST, e_create_date#11438926 ASC NULLS FIRST, e_supplier_id#11438804 ASC NULLS FIRST]
+- WindowGroupLimit [u_brand_context_id#11438504, u_supplier_id#11438570], [similarity#11440052 DESC NULLS LAST, e_create_date#11438926 ASC NULLS FIRST, e_supplier_id#11438804 ASC NULLS FIRST], row_number(), 1, Final
+- *(24) Sort [u_brand_context_id#11438504 ASC NULLS FIRST, u_supplier_id#11438570 ASC NULLS FIRST, similarity#11440052 DESC NULLS LAST, e_create_date#11438926 ASC NULLS FIRST, e_supplier_id#11438804 ASC NULLS FIRST], false, 0
+- AQEShuffleRead coalesced
+- ShuffleQueryStage 26
+- Exchange hashpartitioning(u_brand_context_id#11438504, u_supplier_id#11438570, 10), ENSURE_REQUIREMENTS, [plan_id=584439]
+- WindowGroupLimit [u_brand_context_id#11438504, u_supplier_id#11438570], [similarity#11440052 DESC NULLS LAST, e_create_date#11438926 ASC NULLS FIRST, e_supplier_id#11438804 ASC NULLS FIRST], row_number(), 1, Partial
+- *(23) Sort [u_brand_context_id#11438504 ASC NULLS FIRST, u_supplier_id#11438570 ASC NULLS FIRST, similarity#11440052 DESC NULLS LAST, e_create_date#11438926 ASC NULLS FIRST, e_supplier_id#11438804 ASC NULLS FIRST], false, 0
+- *(23) Project [u_brand_context_id#11438504, u_supplier_id#11438570, e_supplier_id#11438804, e_create_date#11438926, pythonUDF0#11442442 AS similarity#11440052]
+- BatchEvalPython [calculate_similarity(u_normalized_name#11438681, e_normalized_name#11438850)#11440051], [pythonUDF0#11442442]
+- *(22) Project [u_brand_context_id#11438504, u_supplier_id#11438570, u_normalized_name#11438681, e_supplier_id#11438804, e_normalized_name#11438850, e_create_date#11438926]
+- *(22) Filter (cast(pythonUDF0#11442441 as double) >= u_threshold#11438680)
+- BatchEvalPython [calculate_similarity(u_normalized_name#11438681, e_normalized_name#11438850)#11440051], [pythonUDF0#11442441]
+- *(21) Project [u_brand_context_id#11438504, u_supplier_id#11438570, u_threshold#11438680, u_normalized_name#11438681, e_supplier_id#11438804, e_normalized_name#11438850, e_create_date#11438926]
+- *(21) BroadcastHashJoin [u_brand_context_id#11438504, u_location_key#11438790], [e_brand_context_id#11438803, e_location_key#11438896], Inner, BuildLeft, false
:- BroadcastQueryStage 24
: +- BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, true], input[4, string, false]),false), [plan_id=583912]
: +- *(15) Project [brand_context_id#11440635 AS u_brand_context_id#11438504, supplier_id#11440636 AS u_supplier_id#11438570, threshold#11440645 AS u_threshold#11438680, normalized_name#11437594 AS u_normalized_name#11438681, location_key#11437651 AS u_location_key#11438790]
: +- AQEShuffleRead coalesced
: +- ShuffleQueryStage 18
: +- Exchange hashpartitioning(brand_context_id#11440635, supplier_id#11440636, 10), ENSURE_REQUIREMENTS, [plan_id=581371]
: +- *(9) Project [brand_context_id#11440635, supplier_id#11440636, threshold#11440645, pythonUDF0#11442433 AS normalized_name#11437594, concat_ws(|, coalesce(pythonUDF1#11442434, ), coalesce(pythonUDF2#11442435, ), coalesce(pythonUDF3#11442436, )) AS location_key#11437651]
: +- BatchEvalPython [normalize_name(entity_name#11440637)#11437593, normalize_location(city#11440638)#11437606, normalize_location(state#11440639)#11437620, normalize_location(country#11440640)#11437635], [pythonUDF0#11442433, pythonUDF1#11442434, pythonUDF2#11442435, pythonUDF3#11442436]
: +- *(8) Project [brand_context_id#11440635, supplier_id#11440636, entity_name#11440637, city#11440638, state#11440639, country#11440640, threshold#11440645]
: +- *(8) Filter ((isnotnull(pythonUDF0#11442430) AND isnotnull(pythonUDF1#11442431)) AND isnotnull(pythonUDF2#11442432))
: +- BatchEvalPython [normalize_location(city#11440638)#11437606, normalize_location(state#11440639)#11437620, normalize_location(country#11440640)#11437635], [pythonUDF0#11442430, pythonUDF1#11442431, pythonUDF2#11442432]
: +- *(7) Filter ((isnotnull(threshold#11440645) AND isnotnull(brand_context_id#11440635)) AND isnotnull(supplier_id#11440636))
: +- TableCacheQueryStage 9
: +- InMemoryTableScan [brand_context_id#11440635, supplier_id#11440636, entity_name#11440637, city#11440638, state#11440639, country#11440640, threshold#11440645], [isnotnull(threshold#11440645), isnotnull(brand_context_id#11440635), isnotnull(supplier_id#11440636)]
: +- InMemoryRelation [brand_context_id#11440635, supplier_id#11440636, entity_name#11440637, city#11440638, state#11440639, country#11440640, supplier_type#11440641, declaring_supplier_id#11440642, threshold#11440645, create_date#11440646], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- AdaptiveSparkPlan isFinalPlan=false
: +- Project [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, threshold#11412139, create_date#11412140]
: +- Filter isnull(canonical_id#11412137)
: +- InMemoryTableScan [brand_context_id#11412129, canonical_id#11412137, city#11412132, country#11412134, create_date#11412140, declaring_supplier_id#11412136, entity_name#11412131, state#11412133, supplier_id#11412130, supplier_type#11412135, threshold#11412139], [isnull(canonical_id#11412137)]
: +- InMemoryRelation [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, canonical_id#11412137, is_primary#11412138, threshold#11412139, create_date#11412140], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#11412129,supplier_id#11412130,entity_name#11412131,city#11412132,state#11412133,country#11412134,supplier_type#11412135,declaring_supplier_id#11412136,canonical_id#11412137,is_primary#11412138,threshold#11412139,create_date#11412140] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- *(21) Project [brand_context_id#11441081 AS e_brand_context_id#11438803, supplier_id#11441082 AS e_supplier_id#11438804, pythonUDF0#11442437 AS e_normalized_name#11438850, concat_ws(|, coalesce(pythonUDF1#11442438, ), coalesce(pythonUDF2#11442439, ), coalesce(pythonUDF3#11442440, )) AS e_location_key#11438896, create_date#11441092 AS e_create_date#11438926]
+- BatchEvalPython [normalize_name(entity_name#11441083)#11437672, normalize_location(city#11441084)#11437732, normalize_location(state#11441085)#11437750, normalize_location(country#11441086)#11437769], [pythonUDF0#11442437, pythonUDF1#11442438, pythonUDF2#11442439, pythonUDF3#11442440]
+- *(20) Filter isnotnull(brand_context_id#11441081)
+- TableCacheQueryStage 12
+- InMemoryTableScan [brand_context_id#11441081, supplier_id#11441082, entity_name#11441083, city#11441084, state#11441085, country#11441086, create_date#11441092], [isnotnull(brand_context_id#11441081)]
+- InMemoryRelation [brand_context_id#11441081, supplier_id#11441082, canonical_id#11441089, is_primary#11441090, entity_name#11441083, city#11441084, state#11441085, country#11441086, threshold#11441091, create_date#11441092], StorageLevel(disk, memory, deserialized, 1 replicas)
+- AdaptiveSparkPlan isFinalPlan=false
+- Filter isnotnull(canonical_id#11412137)
+- InMemoryTableScan [brand_context_id#11412129, supplier_id#11412130, canonical_id#11412137, is_primary#11412138, entity_name#11412131, city#11412132, state#11412133, country#11412134, threshold#11412139, create_date#11412140], [isnotnull(canonical_id#11412137)]
+- InMemoryRelation [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, canonical_id#11412137, is_primary#11412138, threshold#11412139, create_date#11412140], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#11412129,supplier_id#11412130,entity_name#11412131,city#11412132,state#11412133,country#11412134,supplier_type#11412135,declaring_supplier_id#11412136,canonical_id#11412137,is_primary#11412138,threshold#11412139,create_date#11412140] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Union
:- Project [brand_context_id#11412129, supplier_id#11412130, declaring_supplier_id#11412136, supplier_type#11412135, canonical_id#11438340, 0 AS is_primary#11438410, threshold#11412139 AS configured_threshold#11438411, 100.0 AS match_percentage#11438412, 1.0 AS confidence_score#11438415, exact_id_match AS match_reason#11438416]
: +- BroadcastHashJoin [brand_context_id#11412129, supplier_id#11412130], [brand_context_id#11438332, supplier_id#11438333], Inner, BuildLeft, false
: :- BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, false], input[1, string, false]),false), [plan_id=580135]
: : +- Filter (isnotnull(brand_context_id#11412129) AND isnotnull(supplier_id#11412130))
: : +- InMemoryTableScan [brand_context_id#11412129, supplier_id#11412130, supplier_type#11412135, declaring_supplier_id#11412136, threshold#11412139], [isnotnull(brand_context_id#11412129), isnotnull(supplier_id#11412130)]
: : +- InMemoryRelation [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, threshold#11412139, create_date#11412140], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- AdaptiveSparkPlan isFinalPlan=false
: : +- Project [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, threshold#11412139, create_date#11412140]
: : +- Filter isnull(canonical_id#11412137)
: : +- InMemoryTableScan [brand_context_id#11412129, canonical_id#11412137, city#11412132, country#11412134, create_date#11412140, declaring_supplier_id#11412136, entity_name#11412131, state#11412133, supplier_id#11412130, supplier_type#11412135, threshold#11412139], [isnull(canonical_id#11412137)]
: : +- InMemoryRelation [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, canonical_id#11412137, is_primary#11412138, threshold#11412139, create_date#11412140], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#11412129,supplier_id#11412130,entity_name#11412131,city#11412132,state#11412133,country#11412134,supplier_type#11412135,declaring_supplier_id#11412136,canonical_id#11412137,is_primary#11412138,threshold#11412139,create_date#11412140] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
: +- Project [brand_context_id#11438332, supplier_id#11438333, canonical_id#11438340]
: +- Filter ((((isnotnull(is_primary#11438341) AND is_primary#11438341) AND isnotnull(canonical_id#11438340)) AND isnotnull(brand_context_id#11438332)) AND isnotnull(supplier_id#11438333))
: +- InMemoryTableScan [brand_context_id#11438332, canonical_id#11438340, is_primary#11438341, supplier_id#11438333], [isnotnull(is_primary#11438341), is_primary#11438341, isnotnull(canonical_id#11438340), isnotnull(brand_context_id#11438332), isnotnull(supplier_id#11438333)]
: +- InMemoryRelation [brand_context_id#11438332, supplier_id#11438333, canonical_id#11438340, is_primary#11438341, entity_name#11438334, city#11438335, state#11438336, country#11438337, threshold#11438342, create_date#11438343], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- AdaptiveSparkPlan isFinalPlan=false
: +- Filter isnotnull(canonical_id#11412137)
: +- InMemoryTableScan [brand_context_id#11412129, supplier_id#11412130, canonical_id#11412137, is_primary#11412138, entity_name#11412131, city#11412132, state#11412133, country#11412134, threshold#11412139, create_date#11412140], [isnotnull(canonical_id#11412137)]
: +- InMemoryRelation [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, canonical_id#11412137, is_primary#11412138, threshold#11412139, create_date#11412140], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#11412129,supplier_id#11412130,entity_name#11412131,city#11412132,state#11412133,country#11412134,supplier_type#11412135,declaring_supplier_id#11412136,canonical_id#11412137,is_primary#11412138,threshold#11412139,create_date#11412140] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
:- Project [u_brand_context_id#11438504 AS brand_context_id#11440593, u_supplier_id#11438570 AS supplier_id#11440594, u_declaring_supplier_id#11438676 AS declaring_supplier_id#11440595, u_supplier_type#11438677 AS supplier_type#11440596, e_canonical_id#11438805 AS canonical_id#11440617, 0 AS is_primary#11440618, u_threshold#11438680 AS configured_threshold#11440619, cast((similarity#11440052 * 100.0) as double) AS match_percentage#11440976, cast(similarity#11440052 as double) AS confidence_score#11440977, matched_existing AS match_reason#11440622]
: +- Filter (rank#11440316 = 1)
: +- Window [row_number() windowspecdefinition(u_brand_context_id#11438504, u_supplier_id#11438570, similarity#11440052 DESC NULLS LAST, e_create_date#11438926 ASC NULLS FIRST, e_supplier_id#11438804 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank#11440316], [u_brand_context_id#11438504, u_supplier_id#11438570], [similarity#11440052 DESC NULLS LAST, e_create_date#11438926 ASC NULLS FIRST, e_supplier_id#11438804 ASC NULLS FIRST]
: +- WindowGroupLimit [u_brand_context_id#11438504, u_supplier_id#11438570], [similarity#11440052 DESC NULLS LAST, e_create_date#11438926 ASC NULLS FIRST, e_supplier_id#11438804 ASC NULLS FIRST], row_number(), 1, Final
: +- Sort [u_brand_context_id#11438504 ASC NULLS FIRST, u_supplier_id#11438570 ASC NULLS FIRST, similarity#11440052 DESC NULLS LAST, e_create_date#11438926 ASC NULLS FIRST, e_supplier_id#11438804 ASC NULLS FIRST], false, 0
: +- Exchange hashpartitioning(u_brand_context_id#11438504, u_supplier_id#11438570, 10), ENSURE_REQUIREMENTS, [plan_id=580162]
: +- WindowGroupLimit [u_brand_context_id#11438504, u_supplier_id#11438570], [similarity#11440052 DESC NULLS LAST, e_create_date#11438926 ASC NULLS FIRST, e_supplier_id#11438804 ASC NULLS FIRST], row_number(), 1, Partial
: +- Sort [u_brand_context_id#11438504 ASC NULLS FIRST, u_supplier_id#11438570 ASC NULLS FIRST, similarity#11440052 DESC NULLS LAST, e_create_date#11438926 ASC NULLS FIRST, e_supplier_id#11438804 ASC NULLS FIRST], false, 0
: +- Project [u_brand_context_id#11438504, u_supplier_id#11438570, u_declaring_supplier_id#11438676, u_supplier_type#11438677, u_threshold#11438680, e_supplier_id#11438804, e_canonical_id#11438805, e_create_date#11438926, pythonUDF0#11442429 AS similarity#11440052]
: +- BatchEvalPython [calculate_similarity(u_normalized_name#11438681, e_normalized_name#11438850)#11440051], [pythonUDF0#11442429]
: +- Project [u_brand_context_id#11438504, u_supplier_id#11438570, u_declaring_supplier_id#11438676, u_supplier_type#11438677, u_threshold#11438680, u_normalized_name#11438681, e_supplier_id#11438804, e_canonical_id#11438805, e_normalized_name#11438850, e_create_date#11438926]
: +- Filter (cast(pythonUDF0#11442428 as double) >= u_threshold#11438680)
: +- BatchEvalPython [calculate_similarity(u_normalized_name#11438681, e_normalized_name#11438850)#11440051], [pythonUDF0#11442428]
: +- Project [u_brand_context_id#11438504, u_supplier_id#11438570, u_declaring_supplier_id#11438676, u_supplier_type#11438677, u_threshold#11438680, u_normalized_name#11438681, e_supplier_id#11438804, e_canonical_id#11438805, e_normalized_name#11438850, e_create_date#11438926]
: +- BroadcastHashJoin [u_brand_context_id#11438504, u_location_key#11438790], [e_brand_context_id#11438803, e_location_key#11438896], Inner, BuildLeft, false
: :- BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, true], input[6, string, false]),false), [plan_id=580150]
: : +- Project [brand_context_id#11440771 AS u_brand_context_id#11438504, supplier_id#11440772 AS u_supplier_id#11438570, declaring_supplier_id#11440778 AS u_declaring_supplier_id#11438676, supplier_type#11440777 AS u_supplier_type#11438677, threshold#11440781 AS u_threshold#11438680, normalized_name#11437594 AS u_normalized_name#11438681, location_key#11437651 AS u_location_key#11438790]
: : +- SortMergeJoin [brand_context_id#11440771, supplier_id#11440772], [brand_context_id#11438473, supplier_id#11438474], LeftAnti
: : :- Sort [brand_context_id#11440771 ASC NULLS FIRST, supplier_id#11440772 ASC NULLS FIRST], false, 0
: : : +- Exchange hashpartitioning(brand_context_id#11440771, supplier_id#11440772, 10), ENSURE_REQUIREMENTS, [plan_id=580143]
: : : +- Project [brand_context_id#11440771, supplier_id#11440772, supplier_type#11440777, declaring_supplier_id#11440778, threshold#11440781, pythonUDF0#11442420 AS normalized_name#11437594, concat_ws(|, coalesce(pythonUDF1#11442421, ), coalesce(pythonUDF2#11442422, ), coalesce(pythonUDF3#11442423, )) AS location_key#11437651]
: : : +- BatchEvalPython [normalize_name(entity_name#11440773)#11437593, normalize_location(city#11440774)#11437606, normalize_location(state#11440775)#11437620, normalize_location(country#11440776)#11437635], [pythonUDF0#11442420, pythonUDF1#11442421, pythonUDF2#11442422, pythonUDF3#11442423]
: : : +- Project [brand_context_id#11440771, supplier_id#11440772, entity_name#11440773, city#11440774, state#11440775, country#11440776, supplier_type#11440777, declaring_supplier_id#11440778, threshold#11440781]
: : : +- Filter ((isnotnull(pythonUDF0#11442417) AND isnotnull(pythonUDF1#11442418)) AND isnotnull(pythonUDF2#11442419))
: : : +- BatchEvalPython [normalize_location(city#11440774)#11437606, normalize_location(state#11440775)#11437620, normalize_location(country#11440776)#11437635], [pythonUDF0#11442417, pythonUDF1#11442418, pythonUDF2#11442419]
: : : +- Filter (isnotnull(threshold#11440781) AND isnotnull(brand_context_id#11440771))
: : : +- InMemoryTableScan [brand_context_id#11440771, supplier_id#11440772, entity_name#11440773, city#11440774, state#11440775, country#11440776, supplier_type#11440777, declaring_supplier_id#11440778, threshold#11440781], [isnotnull(threshold#11440781), isnotnull(brand_context_id#11440771)]
: : : +- InMemoryRelation [brand_context_id#11440771, supplier_id#11440772, entity_name#11440773, city#11440774, state#11440775, country#11440776, supplier_type#11440777, declaring_supplier_id#11440778, threshold#11440781, create_date#11440782], StorageLevel(disk, memory, deserialized, 1 replicas)
: : : +- AdaptiveSparkPlan isFinalPlan=false
: : : +- Project [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, threshold#11412139, create_date#11412140]
: : : +- Filter isnull(canonical_id#11412137)
: : : +- InMemoryTableScan [brand_context_id#11412129, canonical_id#11412137, city#11412132, country#11412134, create_date#11412140, declaring_supplier_id#11412136, entity_name#11412131, state#11412133, supplier_id#11412130, supplier_type#11412135, threshold#11412139], [isnull(canonical_id#11412137)]
: : : +- InMemoryRelation [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, canonical_id#11412137, is_primary#11412138, threshold#11412139, create_date#11412140], StorageLevel(disk, memory, deserialized, 1 replicas)
: : : +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#11412129,supplier_id#11412130,entity_name#11412131,city#11412132,state#11412133,country#11412134,supplier_type#11412135,declaring_supplier_id#11412136,canonical_id#11412137,is_primary#11412138,threshold#11412139,create_date#11412140] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
: : +- Sort [brand_context_id#11438473 ASC NULLS FIRST, supplier_id#11438474 ASC NULLS FIRST], false, 0
: : +- Exchange hashpartitioning(brand_context_id#11438473, supplier_id#11438474, 10), ENSURE_REQUIREMENTS, [plan_id=580144]
: : +- Project [brand_context_id#11438473, supplier_id#11438474]
: : +- BroadcastHashJoin [brand_context_id#11438473, supplier_id#11438474], [brand_context_id#11440783, supplier_id#11440784], Inner, BuildLeft, false
: : :- BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, false], input[1, string, false]),false), [plan_id=580138]
: : : +- Filter (isnotnull(brand_context_id#11438473) AND isnotnull(supplier_id#11438474))
: : : +- InMemoryTableScan [brand_context_id#11438473, supplier_id#11438474], [isnotnull(brand_context_id#11438473), isnotnull(supplier_id#11438474)]
: : : +- InMemoryRelation [brand_context_id#11438473, supplier_id#11438474, entity_name#11438475, city#11438476, state#11438477, country#11438478, supplier_type#11438479, declaring_supplier_id#11438480, threshold#11438483, create_date#11438484], StorageLevel(disk, memory, deserialized, 1 replicas)
: : : +- AdaptiveSparkPlan isFinalPlan=false
: : : +- Project [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, threshold#11412139, create_date#11412140]
: : : +- Filter isnull(canonical_id#11412137)
: : : +- InMemoryTableScan [brand_context_id#11412129, canonical_id#11412137, city#11412132, country#11412134, create_date#11412140, declaring_supplier_id#11412136, entity_name#11412131, state#11412133, supplier_id#11412130, supplier_type#11412135, threshold#11412139], [isnull(canonical_id#11412137)]
: : : +- InMemoryRelation [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, canonical_id#11412137, is_primary#11412138, threshold#11412139, create_date#11412140], StorageLevel(disk, memory, deserialized, 1 replicas)
: : : +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#11412129,supplier_id#11412130,entity_name#11412131,city#11412132,state#11412133,country#11412134,supplier_type#11412135,declaring_supplier_id#11412136,canonical_id#11412137,is_primary#11412138,threshold#11412139,create_date#11412140] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
: : +- Project [brand_context_id#11440783, supplier_id#11440784]
: : +- Filter ((((isnotnull(is_primary#11440792) AND is_primary#11440792) AND isnotnull(canonical_id#11440791)) AND isnotnull(brand_context_id#11440783)) AND isnotnull(supplier_id#11440784))
: : +- InMemoryTableScan [brand_context_id#11440783, canonical_id#11440791, is_primary#11440792, supplier_id#11440784], [isnotnull(is_primary#11440792), is_primary#11440792, isnotnull(canonical_id#11440791), isnotnull(brand_context_id#11440783), isnotnull(supplier_id#11440784)]
: : +- InMemoryRelation [brand_context_id#11440783, supplier_id#11440784, canonical_id#11440791, is_primary#11440792, entity_name#11440785, city#11440786, state#11440787, country#11440788, threshold#11440793, create_date#11440794], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- AdaptiveSparkPlan isFinalPlan=false
: : +- Filter isnotnull(canonical_id#11412137)
: : +- InMemoryTableScan [brand_context_id#11412129, supplier_id#11412130, canonical_id#11412137, is_primary#11412138, entity_name#11412131, city#11412132, state#11412133, country#11412134, threshold#11412139, create_date#11412140], [isnotnull(canonical_id#11412137)]
: : +- InMemoryRelation [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, canonical_id#11412137, is_primary#11412138, threshold#11412139, create_date#11412140], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#11412129,supplier_id#11412130,entity_name#11412131,city#11412132,state#11412133,country#11412134,supplier_type#11412135,declaring_supplier_id#11412136,canonical_id#11412137,is_primary#11412138,threshold#11412139,create_date#11412140] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
: +- Project [brand_context_id#11439375 AS e_brand_context_id#11438803, supplier_id#11439376 AS e_supplier_id#11438804, canonical_id#11439383 AS e_canonical_id#11438805, pythonUDF0#11442424 AS e_normalized_name#11438850, concat_ws(|, coalesce(pythonUDF1#11442425, ), coalesce(pythonUDF2#11442426, ), coalesce(pythonUDF3#11442427, )) AS e_location_key#11438896, create_date#11439386 AS e_create_date#11438926]
: +- BatchEvalPython [normalize_name(entity_name#11439377)#11437672, normalize_location(city#11439378)#11437732, normalize_location(state#11439379)#11437750, normalize_location(country#11439380)#11437769], [pythonUDF0#11442424, pythonUDF1#11442425, pythonUDF2#11442426, pythonUDF3#11442427]
: +- Filter isnotnull(brand_context_id#11439375)
: +- InMemoryTableScan [brand_context_id#11439375, supplier_id#11439376, canonical_id#11439383, entity_name#11439377, city#11439378, state#11439379, country#11439380, create_date#11439386], [isnotnull(brand_context_id#11439375)]
: +- InMemoryRelation [brand_context_id#11439375, supplier_id#11439376, canonical_id#11439383, is_primary#11439384, entity_name#11439377, city#11439378, state#11439379, country#11439380, threshold#11439385, create_date#11439386], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- AdaptiveSparkPlan isFinalPlan=false
: +- Filter isnotnull(canonical_id#11412137)
: +- InMemoryTableScan [brand_context_id#11412129, supplier_id#11412130, canonical_id#11412137, is_primary#11412138, entity_name#11412131, city#11412132, state#11412133, country#11412134, threshold#11412139, create_date#11412140], [isnotnull(canonical_id#11412137)]
: +- InMemoryRelation [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, canonical_id#11412137, is_primary#11412138, threshold#11412139, create_date#11412140], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#11412129,supplier_id#11412130,entity_name#11412131,city#11412132,state#11412133,country#11412134,supplier_type#11412135,declaring_supplier_id#11412136,canonical_id#11412137,is_primary#11412138,threshold#11412139,create_date#11412140] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- Project [brand_context_id#11441045, supplier_id#11441046, declaring_supplier_id#11441052, supplier_type#11441051, supplier_id#11441046 AS canonical_id#11440739, 1 AS is_primary#11440756, threshold#11441055 AS configured_threshold#11440757, 0.0 AS match_percentage#11440758, 0.0 AS confidence_score#11440759, new_canonical AS match_reason#11440760]
+- SortMergeJoin [brand_context_id#11441045, supplier_id#11441046], [brand_context_id#11440593, supplier_id#11440594], LeftAnti
:- SortMergeJoin [brand_context_id#11441045, supplier_id#11441046], [brand_context_id#11441057, supplier_id#11441058], LeftAnti
: :- Sort [brand_context_id#11441045 ASC NULLS FIRST, supplier_id#11441046 ASC NULLS FIRST], false, 0
: : +- Exchange hashpartitioning(brand_context_id#11441045, supplier_id#11441046, 10), ENSURE_REQUIREMENTS, [plan_id=580173]
: : +- InMemoryTableScan [brand_context_id#11441045, supplier_id#11441046, supplier_type#11441051, declaring_supplier_id#11441052, threshold#11441055]
: : +- InMemoryRelation [brand_context_id#11441045, supplier_id#11441046, entity_name#11441047, city#11441048, state#11441049, country#11441050, supplier_type#11441051, declaring_supplier_id#11441052, threshold#11441055, create_date#11441056], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- AdaptiveSparkPlan isFinalPlan=false
: : +- Project [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, threshold#11412139, create_date#11412140]
: : +- Filter isnull(canonical_id#11412137)
: : +- InMemoryTableScan [brand_context_id#11412129, canonical_id#11412137, city#11412132, country#11412134, create_date#11412140, declaring_supplier_id#11412136, entity_name#11412131, state#11412133, supplier_id#11412130, supplier_type#11412135, threshold#11412139], [isnull(canonical_id#11412137)]
: : +- InMemoryRelation [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, canonical_id#11412137, is_primary#11412138, threshold#11412139, create_date#11412140], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#11412129,supplier_id#11412130,entity_name#11412131,city#11412132,state#11412133,country#11412134,supplier_type#11412135,declaring_supplier_id#11412136,canonical_id#11412137,is_primary#11412138,threshold#11412139,create_date#11412140] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
: +- Sort [brand_context_id#11441057 ASC NULLS FIRST, supplier_id#11441058 ASC NULLS FIRST], false, 0
: +- Exchange hashpartitioning(brand_context_id#11441057, supplier_id#11441058, 10), ENSURE_REQUIREMENTS, [plan_id=580174]
: +- Project [brand_context_id#11441057, supplier_id#11441058]
: +- BroadcastHashJoin [brand_context_id#11441057, supplier_id#11441058], [brand_context_id#11441069, supplier_id#11441070], Inner, BuildLeft, false
: :- BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, false], input[1, string, false]),false), [plan_id=580168]
: : +- Filter (isnotnull(brand_context_id#11441057) AND isnotnull(supplier_id#11441058))
: : +- InMemoryTableScan [brand_context_id#11441057, supplier_id#11441058], [isnotnull(brand_context_id#11441057), isnotnull(supplier_id#11441058)]
: : +- InMemoryRelation [brand_context_id#11441057, supplier_id#11441058, entity_name#11441059, city#11441060, state#11441061, country#11441062, supplier_type#11441063, declaring_supplier_id#11441064, threshold#11441067, create_date#11441068], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- AdaptiveSparkPlan isFinalPlan=false
: : +- Project [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, threshold#11412139, create_date#11412140]
: : +- Filter isnull(canonical_id#11412137)
: : +- InMemoryTableScan [brand_context_id#11412129, canonical_id#11412137, city#11412132, country#11412134, create_date#11412140, declaring_supplier_id#11412136, entity_name#11412131, state#11412133, supplier_id#11412130, supplier_type#11412135, threshold#11412139], [isnull(canonical_id#11412137)]
: : +- InMemoryRelation [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, canonical_id#11412137, is_primary#11412138, threshold#11412139, create_date#11412140], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#11412129,supplier_id#11412130,entity_name#11412131,city#11412132,state#11412133,country#11412134,supplier_type#11412135,declaring_supplier_id#11412136,canonical_id#11412137,is_primary#11412138,threshold#11412139,create_date#11412140] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
: +- Project [brand_context_id#11441069, supplier_id#11441070]
: +- Filter ((((isnotnull(is_primary#11441078) AND is_primary#11441078) AND isnotnull(canonical_id#11441077)) AND isnotnull(brand_context_id#11441069)) AND isnotnull(supplier_id#11441070))
: +- InMemoryTableScan [brand_context_id#11441069, canonical_id#11441077, is_primary#11441078, supplier_id#11441070], [isnotnull(is_primary#11441078), is_primary#11441078, isnotnull(canonical_id#11441077), isnotnull(brand_context_id#11441069), isnotnull(supplier_id#11441070)]
: +- InMemoryRelation [brand_context_id#11441069, supplier_id#11441070, canonical_id#11441077, is_primary#11441078, entity_name#11441071, city#11441072, state#11441073, country#11441074, threshold#11441079, create_date#11441080], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- AdaptiveSparkPlan isFinalPlan=false
: +- Filter isnotnull(canonical_id#11412137)
: +- InMemoryTableScan [brand_context_id#11412129, supplier_id#11412130, canonical_id#11412137, is_primary#11412138, entity_name#11412131, city#11412132, state#11412133, country#11412134, threshold#11412139, create_date#11412140], [isnotnull(canonical_id#11412137)]
: +- InMemoryRelation [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, canonical_id#11412137, is_primary#11412138, threshold#11412139, create_date#11412140], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#11412129,supplier_id#11412130,entity_name#11412131,city#11412132,state#11412133,country#11412134,supplier_type#11412135,declaring_supplier_id#11412136,canonical_id#11412137,is_primary#11412138,threshold#11412139,create_date#11412140] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- Project [u_brand_context_id#11438504 AS brand_context_id#11440593, u_supplier_id#11438570 AS supplier_id#11440594]
+- Filter (rank#11440316 = 1)
+- Window [row_number() windowspecdefinition(u_brand_context_id#11438504, u_supplier_id#11438570, similarity#11440052 DESC NULLS LAST, e_create_date#11438926 ASC NULLS FIRST, e_supplier_id#11438804 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank#11440316], [u_brand_context_id#11438504, u_supplier_id#11438570], [similarity#11440052 DESC NULLS LAST, e_create_date#11438926 ASC NULLS FIRST, e_supplier_id#11438804 ASC NULLS FIRST]
+- WindowGroupLimit [u_brand_context_id#11438504, u_supplier_id#11438570], [similarity#11440052 DESC NULLS LAST, e_create_date#11438926 ASC NULLS FIRST, e_supplier_id#11438804 ASC NULLS FIRST], row_number(), 1, Final
+- Sort [u_brand_context_id#11438504 ASC NULLS FIRST, u_supplier_id#11438570 ASC NULLS FIRST, similarity#11440052 DESC NULLS LAST, e_create_date#11438926 ASC NULLS FIRST, e_supplier_id#11438804 ASC NULLS FIRST], false, 0
+- Exchange hashpartitioning(u_brand_context_id#11438504, u_supplier_id#11438570, 10), ENSURE_REQUIREMENTS, [plan_id=580202]
+- WindowGroupLimit [u_brand_context_id#11438504, u_supplier_id#11438570], [similarity#11440052 DESC NULLS LAST, e_create_date#11438926 ASC NULLS FIRST, e_supplier_id#11438804 ASC NULLS FIRST], row_number(), 1, Partial
+- Sort [u_brand_context_id#11438504 ASC NULLS FIRST, u_supplier_id#11438570 ASC NULLS FIRST, similarity#11440052 DESC NULLS LAST, e_create_date#11438926 ASC NULLS FIRST, e_supplier_id#11438804 ASC NULLS FIRST], false, 0
+- Project [u_brand_context_id#11438504, u_supplier_id#11438570, e_supplier_id#11438804, e_create_date#11438926, pythonUDF0#11442442 AS similarity#11440052]
+- BatchEvalPython [calculate_similarity(u_normalized_name#11438681, e_normalized_name#11438850)#11440051], [pythonUDF0#11442442]
+- Project [u_brand_context_id#11438504, u_supplier_id#11438570, u_normalized_name#11438681, e_supplier_id#11438804, e_normalized_name#11438850, e_create_date#11438926]
+- Filter (cast(pythonUDF0#11442441 as double) >= u_threshold#11438680)
+- BatchEvalPython [calculate_similarity(u_normalized_name#11438681, e_normalized_name#11438850)#11440051], [pythonUDF0#11442441]
+- Project [u_brand_context_id#11438504, u_supplier_id#11438570, u_threshold#11438680, u_normalized_name#11438681, e_supplier_id#11438804, e_normalized_name#11438850, e_create_date#11438926]
+- BroadcastHashJoin [u_brand_context_id#11438504, u_location_key#11438790], [e_brand_context_id#11438803, e_location_key#11438896], Inner, BuildLeft, false
:- BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, true], input[4, string, false]),false), [plan_id=580190]
: +- Project [brand_context_id#11440635 AS u_brand_context_id#11438504, supplier_id#11440636 AS u_supplier_id#11438570, threshold#11440645 AS u_threshold#11438680, normalized_name#11437594 AS u_normalized_name#11438681, location_key#11437651 AS u_location_key#11438790]
: +- SortMergeJoin [brand_context_id#11440635, supplier_id#11440636], [brand_context_id#11440647, supplier_id#11440648], LeftAnti
: :- Sort [brand_context_id#11440635 ASC NULLS FIRST, supplier_id#11440636 ASC NULLS FIRST], false, 0
: : +- Exchange hashpartitioning(brand_context_id#11440635, supplier_id#11440636, 10), ENSURE_REQUIREMENTS, [plan_id=580183]
: : +- Project [brand_context_id#11440635, supplier_id#11440636, threshold#11440645, pythonUDF0#11442433 AS normalized_name#11437594, concat_ws(|, coalesce(pythonUDF1#11442434, ), coalesce(pythonUDF2#11442435, ), coalesce(pythonUDF3#11442436, )) AS location_key#11437651]
: : +- BatchEvalPython [normalize_name(entity_name#11440637)#11437593, normalize_location(city#11440638)#11437606, normalize_location(state#11440639)#11437620, normalize_location(country#11440640)#11437635], [pythonUDF0#11442433, pythonUDF1#11442434, pythonUDF2#11442435, pythonUDF3#11442436]
: : +- Project [brand_context_id#11440635, supplier_id#11440636, entity_name#11440637, city#11440638, state#11440639, country#11440640, threshold#11440645]
: : +- Filter ((isnotnull(pythonUDF0#11442430) AND isnotnull(pythonUDF1#11442431)) AND isnotnull(pythonUDF2#11442432))
: : +- BatchEvalPython [normalize_location(city#11440638)#11437606, normalize_location(state#11440639)#11437620, normalize_location(country#11440640)#11437635], [pythonUDF0#11442430, pythonUDF1#11442431, pythonUDF2#11442432]
: : +- Filter ((isnotnull(threshold#11440645) AND isnotnull(brand_context_id#11440635)) AND isnotnull(supplier_id#11440636))
: : +- InMemoryTableScan [brand_context_id#11440635, supplier_id#11440636, entity_name#11440637, city#11440638, state#11440639, country#11440640, threshold#11440645], [isnotnull(threshold#11440645), isnotnull(brand_context_id#11440635), isnotnull(supplier_id#11440636)]
: : +- InMemoryRelation [brand_context_id#11440635, supplier_id#11440636, entity_name#11440637, city#11440638, state#11440639, country#11440640, supplier_type#11440641, declaring_supplier_id#11440642, threshold#11440645, create_date#11440646], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- AdaptiveSparkPlan isFinalPlan=false
: : +- Project [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, threshold#11412139, create_date#11412140]
: : +- Filter isnull(canonical_id#11412137)
: : +- InMemoryTableScan [brand_context_id#11412129, canonical_id#11412137, city#11412132, country#11412134, create_date#11412140, declaring_supplier_id#11412136, entity_name#11412131, state#11412133, supplier_id#11412130, supplier_type#11412135, threshold#11412139], [isnull(canonical_id#11412137)]
: : +- InMemoryRelation [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, canonical_id#11412137, is_primary#11412138, threshold#11412139, create_date#11412140], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#11412129,supplier_id#11412130,entity_name#11412131,city#11412132,state#11412133,country#11412134,supplier_type#11412135,declaring_supplier_id#11412136,canonical_id#11412137,is_primary#11412138,threshold#11412139,create_date#11412140] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
: +- Sort [brand_context_id#11440647 ASC NULLS FIRST, supplier_id#11440648 ASC NULLS FIRST], false, 0
: +- Exchange hashpartitioning(brand_context_id#11440647, supplier_id#11440648, 10), ENSURE_REQUIREMENTS, [plan_id=580184]
: +- Project [brand_context_id#11440647, supplier_id#11440648]
: +- BroadcastHashJoin [brand_context_id#11440647, supplier_id#11440648], [brand_context_id#11440659, supplier_id#11440660], Inner, BuildLeft, false
: :- BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, false], input[1, string, false]),false), [plan_id=580178]
: : +- Filter (isnotnull(brand_context_id#11440647) AND isnotnull(supplier_id#11440648))
: : +- InMemoryTableScan [brand_context_id#11440647, supplier_id#11440648], [isnotnull(brand_context_id#11440647), isnotnull(supplier_id#11440648)]
: : +- InMemoryRelation [brand_context_id#11440647, supplier_id#11440648, entity_name#11440649, city#11440650, state#11440651, country#11440652, supplier_type#11440653, declaring_supplier_id#11440654, threshold#11440657, create_date#11440658], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- AdaptiveSparkPlan isFinalPlan=false
: : +- Project [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, threshold#11412139, create_date#11412140]
: : +- Filter isnull(canonical_id#11412137)
: : +- InMemoryTableScan [brand_context_id#11412129, canonical_id#11412137, city#11412132, country#11412134, create_date#11412140, declaring_supplier_id#11412136, entity_name#11412131, state#11412133, supplier_id#11412130, supplier_type#11412135, threshold#11412139], [isnull(canonical_id#11412137)]
: : +- InMemoryRelation [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, canonical_id#11412137, is_primary#11412138, threshold#11412139, create_date#11412140], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#11412129,supplier_id#11412130,entity_name#11412131,city#11412132,state#11412133,country#11412134,supplier_type#11412135,declaring_supplier_id#11412136,canonical_id#11412137,is_primary#11412138,threshold#11412139,create_date#11412140] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
: +- Project [brand_context_id#11440659, supplier_id#11440660]
: +- Filter ((((isnotnull(is_primary#11440668) AND is_primary#11440668) AND isnotnull(canonical_id#11440667)) AND isnotnull(brand_context_id#11440659)) AND isnotnull(supplier_id#11440660))
: +- InMemoryTableScan [brand_context_id#11440659, canonical_id#11440667, is_primary#11440668, supplier_id#11440660], [isnotnull(is_primary#11440668), is_primary#11440668, isnotnull(canonical_id#11440667), isnotnull(brand_context_id#11440659), isnotnull(supplier_id#11440660)]
: +- InMemoryRelation [brand_context_id#11440659, supplier_id#11440660, canonical_id#11440667, is_primary#11440668, entity_name#11440661, city#11440662, state#11440663, country#11440664, threshold#11440669, create_date#11440670], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- AdaptiveSparkPlan isFinalPlan=false
: +- Filter isnotnull(canonical_id#11412137)
: +- InMemoryTableScan [brand_context_id#11412129, supplier_id#11412130, canonical_id#11412137, is_primary#11412138, entity_name#11412131, city#11412132, state#11412133, country#11412134, threshold#11412139, create_date#11412140], [isnotnull(canonical_id#11412137)]
: +- InMemoryRelation [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, canonical_id#11412137, is_primary#11412138, threshold#11412139, create_date#11412140], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#11412129,supplier_id#11412130,entity_name#11412131,city#11412132,state#11412133,country#11412134,supplier_type#11412135,declaring_supplier_id#11412136,canonical_id#11412137,is_primary#11412138,threshold#11412139,create_date#11412140] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- Project [brand_context_id#11441081 AS e_brand_context_id#11438803, supplier_id#11441082 AS e_supplier_id#11438804, pythonUDF0#11442437 AS e_normalized_name#11438850, concat_ws(|, coalesce(pythonUDF1#11442438, ), coalesce(pythonUDF2#11442439, ), coalesce(pythonUDF3#11442440, )) AS e_location_key#11438896, create_date#11441092 AS e_create_date#11438926]
+- BatchEvalPython [normalize_name(entity_name#11441083)#11437672, normalize_location(city#11441084)#11437732, normalize_location(state#11441085)#11437750, normalize_location(country#11441086)#11437769], [pythonUDF0#11442437, pythonUDF1#11442438, pythonUDF2#11442439, pythonUDF3#11442440]
+- Filter isnotnull(brand_context_id#11441081)
+- InMemoryTableScan [brand_context_id#11441081, supplier_id#11441082, entity_name#11441083, city#11441084, state#11441085, country#11441086, create_date#11441092], [isnotnull(brand_context_id#11441081)]
+- InMemoryRelation [brand_context_id#11441081, supplier_id#11441082, canonical_id#11441089, is_primary#11441090, entity_name#11441083, city#11441084, state#11441085, country#11441086, threshold#11441091, create_date#11441092], StorageLevel(disk, memory, deserialized, 1 replicas)
+- AdaptiveSparkPlan isFinalPlan=false
+- Filter isnotnull(canonical_id#11412137)
+- InMemoryTableScan [brand_context_id#11412129, supplier_id#11412130, canonical_id#11412137, is_primary#11412138, entity_name#11412131, city#11412132, state#11412133, country#11412134, threshold#11412139, create_date#11412140], [isnotnull(canonical_id#11412137)]
+- InMemoryRelation [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, canonical_id#11412137, is_primary#11412138, threshold#11412139, create_date#11412140], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#11412129,supplier_id#11412130,entity_name#11412131,city#11412132,state#11412133,country#11412134,supplier_type#11412135,declaring_supplier_id#11412136,canonical_id#11412137,is_primary#11412138,threshold#11412139,create_date#11412140] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(3) InMemoryTableScan
Output [5]: [brand_context_id#11412129, supplier_id#11412130, supplier_type#11412135, declaring_supplier_id#11412136, threshold#11412139]
Arguments: [brand_context_id#11412129, supplier_id#11412130, supplier_type#11412135, declaring_supplier_id#11412136, threshold#11412139], [isnotnull(brand_context_id#11412129), isnotnull(supplier_id#11412130)]
(4) InMemoryRelation
Arguments: [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, threshold#11412139, create_date#11412140], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@7795a5ad,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Project [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, threshold#11412139, create_date#11412140]
+- *(1) Filter isnull(canonical_id#11412137)
+- TableCacheQueryStage 0
+- InMemoryTableScan [brand_context_id#11412129, canonical_id#11412137, city#11412132, country#11412134, create_date#11412140, declaring_supplier_id#11412136, entity_name#11412131, state#11412133, supplier_id#11412130, supplier_type#11412135, threshold#11412139], [isnull(canonical_id#11412137)]
+- InMemoryRelation [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, canonical_id#11412137, is_primary#11412138, threshold#11412139, create_date#11412140], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#11412129,supplier_id#11412130,entity_name#11412131,city#11412132,state#11412133,country#11412134,supplier_type#11412135,declaring_supplier_id#11412136,canonical_id#11412137,is_primary#11412138,threshold#11412139,create_date#11412140] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Project [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, threshold#11412139, create_date#11412140]
+- Filter isnull(canonical_id#11412137)
+- InMemoryTableScan [brand_context_id#11412129, canonical_id#11412137, city#11412132, country#11412134, create_date#11412140, declaring_supplier_id#11412136, entity_name#11412131, state#11412133, supplier_id#11412130, supplier_type#11412135, threshold#11412139], [isnull(canonical_id#11412137)]
+- InMemoryRelation [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, canonical_id#11412137, is_primary#11412138, threshold#11412139, create_date#11412140], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#11412129,supplier_id#11412130,entity_name#11412131,city#11412132,state#11412133,country#11412134,supplier_type#11412135,declaring_supplier_id#11412136,canonical_id#11412137,is_primary#11412138,threshold#11412139,create_date#11412140] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(5) InMemoryTableScan
Output [11]: [brand_context_id#11412129, canonical_id#11412137, city#11412132, country#11412134, create_date#11412140, declaring_supplier_id#11412136, entity_name#11412131, state#11412133, supplier_id#11412130, supplier_type#11412135, threshold#11412139]
Arguments: [brand_context_id#11412129, canonical_id#11412137, city#11412132, country#11412134, create_date#11412140, declaring_supplier_id#11412136, entity_name#11412131, state#11412133, supplier_id#11412130, supplier_type#11412135, threshold#11412139], [isnull(canonical_id#11412137)]
(6) InMemoryRelation
Arguments: [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, canonical_id#11412137, is_primary#11412138, threshold#11412139, create_date#11412140], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@7795a5ad,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#11412129,supplier_id#11412130,entity_name#11412131,city#11412132,state#11412133,country#11412134,supplier_type#11412135,declaring_supplier_id#11412136,canonical_id#11412137,is_primary#11412138,threshold#11412139,create_date#11412140] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(7) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [codegen id : 1]
Output [12]: [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, canonical_id#11412137, is_primary#11412138, threshold#11412139, create_date#11412140]
ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,country:string,supplier_type:string,declaring_supplier_id:string,canonical_id:string,is_primary:boolean,threshold:double,create_date:date>
(8) Filter
Input [11]: [brand_context_id#11412129, canonical_id#11412137, city#11412132, country#11412134, create_date#11412140, declaring_supplier_id#11412136, entity_name#11412131, state#11412133, supplier_id#11412130, supplier_type#11412135, threshold#11412139]
Condition : isnull(canonical_id#11412137)
(9) Project
Output [10]: [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, threshold#11412139, create_date#11412140]
Input [11]: [brand_context_id#11412129, canonical_id#11412137, city#11412132, country#11412134, create_date#11412140, declaring_supplier_id#11412136, entity_name#11412131, state#11412133, supplier_id#11412130, supplier_type#11412135, threshold#11412139]
(10) AdaptiveSparkPlan
Output [10]: [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, threshold#11412139, create_date#11412140]
Arguments: isFinalPlan=false
(11) Filter
Input [5]: [brand_context_id#11412129, supplier_id#11412130, supplier_type#11412135, declaring_supplier_id#11412136, threshold#11412139]
Condition : (isnotnull(brand_context_id#11412129) AND isnotnull(supplier_id#11412130))
(12) BroadcastExchange
Input [5]: [brand_context_id#11412129, supplier_id#11412130, supplier_type#11412135, declaring_supplier_id#11412136, threshold#11412139]
Arguments: HashedRelationBroadcastMode(List(input[0, string, false], input[1, string, false]),false), [plan_id=587855]
(13) InMemoryTableScan
Output [4]: [brand_context_id#11438332, canonical_id#11438340, is_primary#11438341, supplier_id#11438333]
Arguments: [brand_context_id#11438332, canonical_id#11438340, is_primary#11438341, supplier_id#11438333], [isnotnull(is_primary#11438341), is_primary#11438341, isnotnull(canonical_id#11438340), isnotnull(brand_context_id#11438332), isnotnull(supplier_id#11438333)]
(14) InMemoryRelation
Arguments: [brand_context_id#11438332, supplier_id#11438333, canonical_id#11438340, is_primary#11438341, entity_name#11438334, city#11438335, state#11438336, country#11438337, threshold#11438342, create_date#11438343], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@7795a5ad,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Filter isnotnull(canonical_id#11412137)
+- TableCacheQueryStage 0
+- InMemoryTableScan [brand_context_id#11412129, supplier_id#11412130, canonical_id#11412137, is_primary#11412138, entity_name#11412131, city#11412132, state#11412133, country#11412134, threshold#11412139, create_date#11412140], [isnotnull(canonical_id#11412137)]
+- InMemoryRelation [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, canonical_id#11412137, is_primary#11412138, threshold#11412139, create_date#11412140], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#11412129,supplier_id#11412130,entity_name#11412131,city#11412132,state#11412133,country#11412134,supplier_type#11412135,declaring_supplier_id#11412136,canonical_id#11412137,is_primary#11412138,threshold#11412139,create_date#11412140] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Filter isnotnull(canonical_id#11412137)
+- InMemoryTableScan [brand_context_id#11412129, supplier_id#11412130, canonical_id#11412137, is_primary#11412138, entity_name#11412131, city#11412132, state#11412133, country#11412134, threshold#11412139, create_date#11412140], [isnotnull(canonical_id#11412137)]
+- InMemoryRelation [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, canonical_id#11412137, is_primary#11412138, threshold#11412139, create_date#11412140], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#11412129,supplier_id#11412130,entity_name#11412131,city#11412132,state#11412133,country#11412134,supplier_type#11412135,declaring_supplier_id#11412136,canonical_id#11412137,is_primary#11412138,threshold#11412139,create_date#11412140] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(15) InMemoryTableScan
Output [10]: [brand_context_id#11412129, supplier_id#11412130, canonical_id#11412137, is_primary#11412138, entity_name#11412131, city#11412132, state#11412133, country#11412134, threshold#11412139, create_date#11412140]
Arguments: [brand_context_id#11412129, supplier_id#11412130, canonical_id#11412137, is_primary#11412138, entity_name#11412131, city#11412132, state#11412133, country#11412134, threshold#11412139, create_date#11412140], [isnotnull(canonical_id#11412137)]
(16) InMemoryRelation
Arguments: [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, canonical_id#11412137, is_primary#11412138, threshold#11412139, create_date#11412140], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@7795a5ad,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#11412129,supplier_id#11412130,entity_name#11412131,city#11412132,state#11412133,country#11412134,supplier_type#11412135,declaring_supplier_id#11412136,canonical_id#11412137,is_primary#11412138,threshold#11412139,create_date#11412140] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(17) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [codegen id : 1]
Output [12]: [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, canonical_id#11412137, is_primary#11412138, threshold#11412139, create_date#11412140]
ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,country:string,supplier_type:string,declaring_supplier_id:string,canonical_id:string,is_primary:boolean,threshold:double,create_date:date>
(18) Filter
Input [10]: [brand_context_id#11412129, supplier_id#11412130, canonical_id#11412137, is_primary#11412138, entity_name#11412131, city#11412132, state#11412133, country#11412134, threshold#11412139, create_date#11412140]
Condition : isnotnull(canonical_id#11412137)
(19) AdaptiveSparkPlan
Output [10]: [brand_context_id#11412129, supplier_id#11412130, canonical_id#11412137, is_primary#11412138, entity_name#11412131, city#11412132, state#11412133, country#11412134, threshold#11412139, create_date#11412140]
Arguments: isFinalPlan=false
(20) Filter
Input [4]: [brand_context_id#11438332, canonical_id#11438340, is_primary#11438341, supplier_id#11438333]
Condition : ((((isnotnull(is_primary#11438341) AND is_primary#11438341) AND isnotnull(canonical_id#11438340)) AND isnotnull(brand_context_id#11438332)) AND isnotnull(supplier_id#11438333))
(21) Project
Output [3]: [brand_context_id#11438332, supplier_id#11438333, canonical_id#11438340]
Input [4]: [brand_context_id#11438332, canonical_id#11438340, is_primary#11438341, supplier_id#11438333]
(22) BroadcastHashJoin
Left keys [2]: [brand_context_id#11412129, supplier_id#11412130]
Right keys [2]: [brand_context_id#11438332, supplier_id#11438333]
Join type: Inner
Join condition: None
(23) Project
Output [10]: [brand_context_id#11412129, supplier_id#11412130, declaring_supplier_id#11412136, supplier_type#11412135, canonical_id#11438340, 0 AS is_primary#11438410, threshold#11412139 AS configured_threshold#11438411, 100.0 AS match_percentage#11438412, 1.0 AS confidence_score#11438415, exact_id_match AS match_reason#11438416]
Input [8]: [brand_context_id#11412129, supplier_id#11412130, supplier_type#11412135, declaring_supplier_id#11412136, threshold#11412139, brand_context_id#11438332, supplier_id#11438333, canonical_id#11438340]
(24) InMemoryTableScan
Output [9]: [brand_context_id#11440771, supplier_id#11440772, entity_name#11440773, city#11440774, state#11440775, country#11440776, supplier_type#11440777, declaring_supplier_id#11440778, threshold#11440781]
Arguments: [brand_context_id#11440771, supplier_id#11440772, entity_name#11440773, city#11440774, state#11440775, country#11440776, supplier_type#11440777, declaring_supplier_id#11440778, threshold#11440781], [isnotnull(threshold#11440781), isnotnull(brand_context_id#11440771)]
(25) InMemoryRelation
Arguments: [brand_context_id#11440771, supplier_id#11440772, entity_name#11440773, city#11440774, state#11440775, country#11440776, supplier_type#11440777, declaring_supplier_id#11440778, threshold#11440781, create_date#11440782], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@7795a5ad,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Project [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, threshold#11412139, create_date#11412140]
+- *(1) Filter isnull(canonical_id#11412137)
+- TableCacheQueryStage 0
+- InMemoryTableScan [brand_context_id#11412129, canonical_id#11412137, city#11412132, country#11412134, create_date#11412140, declaring_supplier_id#11412136, entity_name#11412131, state#11412133, supplier_id#11412130, supplier_type#11412135, threshold#11412139], [isnull(canonical_id#11412137)]
+- InMemoryRelation [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, canonical_id#11412137, is_primary#11412138, threshold#11412139, create_date#11412140], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#11412129,supplier_id#11412130,entity_name#11412131,city#11412132,state#11412133,country#11412134,supplier_type#11412135,declaring_supplier_id#11412136,canonical_id#11412137,is_primary#11412138,threshold#11412139,create_date#11412140] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Project [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, threshold#11412139, create_date#11412140]
+- Filter isnull(canonical_id#11412137)
+- InMemoryTableScan [brand_context_id#11412129, canonical_id#11412137, city#11412132, country#11412134, create_date#11412140, declaring_supplier_id#11412136, entity_name#11412131, state#11412133, supplier_id#11412130, supplier_type#11412135, threshold#11412139], [isnull(canonical_id#11412137)]
+- InMemoryRelation [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, canonical_id#11412137, is_primary#11412138, threshold#11412139, create_date#11412140], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#11412129,supplier_id#11412130,entity_name#11412131,city#11412132,state#11412133,country#11412134,supplier_type#11412135,declaring_supplier_id#11412136,canonical_id#11412137,is_primary#11412138,threshold#11412139,create_date#11412140] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(26) AdaptiveSparkPlan
Output [10]: [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, threshold#11412139, create_date#11412140]
Arguments: isFinalPlan=false
(27) Filter
Input [9]: [brand_context_id#11440771, supplier_id#11440772, entity_name#11440773, city#11440774, state#11440775, country#11440776, supplier_type#11440777, declaring_supplier_id#11440778, threshold#11440781]
Condition : (isnotnull(threshold#11440781) AND isnotnull(brand_context_id#11440771))
(28) BatchEvalPython
Input [9]: [brand_context_id#11440771, supplier_id#11440772, entity_name#11440773, city#11440774, state#11440775, country#11440776, supplier_type#11440777, declaring_supplier_id#11440778, threshold#11440781]
Arguments: [normalize_location(city#11440774)#11437606, normalize_location(state#11440775)#11437620, normalize_location(country#11440776)#11437635], [pythonUDF0#11442417, pythonUDF1#11442418, pythonUDF2#11442419]
(29) Filter
Input [12]: [brand_context_id#11440771, supplier_id#11440772, entity_name#11440773, city#11440774, state#11440775, country#11440776, supplier_type#11440777, declaring_supplier_id#11440778, threshold#11440781, pythonUDF0#11442417, pythonUDF1#11442418, pythonUDF2#11442419]
Condition : ((isnotnull(pythonUDF0#11442417) AND isnotnull(pythonUDF1#11442418)) AND isnotnull(pythonUDF2#11442419))
(30) Project
Output [9]: [brand_context_id#11440771, supplier_id#11440772, entity_name#11440773, city#11440774, state#11440775, country#11440776, supplier_type#11440777, declaring_supplier_id#11440778, threshold#11440781]
Input [12]: [brand_context_id#11440771, supplier_id#11440772, entity_name#11440773, city#11440774, state#11440775, country#11440776, supplier_type#11440777, declaring_supplier_id#11440778, threshold#11440781, pythonUDF0#11442417, pythonUDF1#11442418, pythonUDF2#11442419]
(31) BatchEvalPython
Input [9]: [brand_context_id#11440771, supplier_id#11440772, entity_name#11440773, city#11440774, state#11440775, country#11440776, supplier_type#11440777, declaring_supplier_id#11440778, threshold#11440781]
Arguments: [normalize_name(entity_name#11440773)#11437593, normalize_location(city#11440774)#11437606, normalize_location(state#11440775)#11437620, normalize_location(country#11440776)#11437635], [pythonUDF0#11442420, pythonUDF1#11442421, pythonUDF2#11442422, pythonUDF3#11442423]
(32) Project
Output [7]: [brand_context_id#11440771, supplier_id#11440772, supplier_type#11440777, declaring_supplier_id#11440778, threshold#11440781, pythonUDF0#11442420 AS normalized_name#11437594, concat_ws(|, coalesce(pythonUDF1#11442421, ), coalesce(pythonUDF2#11442422, ), coalesce(pythonUDF3#11442423, )) AS location_key#11437651]
Input [13]: [brand_context_id#11440771, supplier_id#11440772, entity_name#11440773, city#11440774, state#11440775, country#11440776, supplier_type#11440777, declaring_supplier_id#11440778, threshold#11440781, pythonUDF0#11442420, pythonUDF1#11442421, pythonUDF2#11442422, pythonUDF3#11442423]
(33) Exchange
Input [7]: [brand_context_id#11440771, supplier_id#11440772, supplier_type#11440777, declaring_supplier_id#11440778, threshold#11440781, normalized_name#11437594, location_key#11437651]
Arguments: hashpartitioning(brand_context_id#11440771, supplier_id#11440772, 10), ENSURE_REQUIREMENTS, [plan_id=587863]
(34) Sort
Input [7]: [brand_context_id#11440771, supplier_id#11440772, supplier_type#11440777, declaring_supplier_id#11440778, threshold#11440781, normalized_name#11437594, location_key#11437651]
Arguments: [brand_context_id#11440771 ASC NULLS FIRST, supplier_id#11440772 ASC NULLS FIRST], false, 0
(35) InMemoryTableScan
Output [2]: [brand_context_id#11438473, supplier_id#11438474]
Arguments: [brand_context_id#11438473, supplier_id#11438474], [isnotnull(brand_context_id#11438473), isnotnull(supplier_id#11438474)]
(36) InMemoryRelation
Arguments: [brand_context_id#11438473, supplier_id#11438474, entity_name#11438475, city#11438476, state#11438477, country#11438478, supplier_type#11438479, declaring_supplier_id#11438480, threshold#11438483, create_date#11438484], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@7795a5ad,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Project [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, threshold#11412139, create_date#11412140]
+- *(1) Filter isnull(canonical_id#11412137)
+- TableCacheQueryStage 0
+- InMemoryTableScan [brand_context_id#11412129, canonical_id#11412137, city#11412132, country#11412134, create_date#11412140, declaring_supplier_id#11412136, entity_name#11412131, state#11412133, supplier_id#11412130, supplier_type#11412135, threshold#11412139], [isnull(canonical_id#11412137)]
+- InMemoryRelation [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, canonical_id#11412137, is_primary#11412138, threshold#11412139, create_date#11412140], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#11412129,supplier_id#11412130,entity_name#11412131,city#11412132,state#11412133,country#11412134,supplier_type#11412135,declaring_supplier_id#11412136,canonical_id#11412137,is_primary#11412138,threshold#11412139,create_date#11412140] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Project [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, threshold#11412139, create_date#11412140]
+- Filter isnull(canonical_id#11412137)
+- InMemoryTableScan [brand_context_id#11412129, canonical_id#11412137, city#11412132, country#11412134, create_date#11412140, declaring_supplier_id#11412136, entity_name#11412131, state#11412133, supplier_id#11412130, supplier_type#11412135, threshold#11412139], [isnull(canonical_id#11412137)]
+- InMemoryRelation [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, canonical_id#11412137, is_primary#11412138, threshold#11412139, create_date#11412140], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#11412129,supplier_id#11412130,entity_name#11412131,city#11412132,state#11412133,country#11412134,supplier_type#11412135,declaring_supplier_id#11412136,canonical_id#11412137,is_primary#11412138,threshold#11412139,create_date#11412140] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(37) AdaptiveSparkPlan
Output [10]: [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, threshold#11412139, create_date#11412140]
Arguments: isFinalPlan=false
(38) Filter
Input [2]: [brand_context_id#11438473, supplier_id#11438474]
Condition : (isnotnull(brand_context_id#11438473) AND isnotnull(supplier_id#11438474))
(39) BroadcastExchange
Input [2]: [brand_context_id#11438473, supplier_id#11438474]
Arguments: HashedRelationBroadcastMode(List(input[0, string, false], input[1, string, false]),false), [plan_id=587858]
(40) InMemoryTableScan
Output [4]: [brand_context_id#11440783, canonical_id#11440791, is_primary#11440792, supplier_id#11440784]
Arguments: [brand_context_id#11440783, canonical_id#11440791, is_primary#11440792, supplier_id#11440784], [isnotnull(is_primary#11440792), is_primary#11440792, isnotnull(canonical_id#11440791), isnotnull(brand_context_id#11440783), isnotnull(supplier_id#11440784)]
(41) InMemoryRelation
Arguments: [brand_context_id#11440783, supplier_id#11440784, canonical_id#11440791, is_primary#11440792, entity_name#11440785, city#11440786, state#11440787, country#11440788, threshold#11440793, create_date#11440794], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@7795a5ad,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Filter isnotnull(canonical_id#11412137)
+- TableCacheQueryStage 0
+- InMemoryTableScan [brand_context_id#11412129, supplier_id#11412130, canonical_id#11412137, is_primary#11412138, entity_name#11412131, city#11412132, state#11412133, country#11412134, threshold#11412139, create_date#11412140], [isnotnull(canonical_id#11412137)]
+- InMemoryRelation [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, canonical_id#11412137, is_primary#11412138, threshold#11412139, create_date#11412140], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#11412129,supplier_id#11412130,entity_name#11412131,city#11412132,state#11412133,country#11412134,supplier_type#11412135,declaring_supplier_id#11412136,canonical_id#11412137,is_primary#11412138,threshold#11412139,create_date#11412140] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Filter isnotnull(canonical_id#11412137)
+- InMemoryTableScan [brand_context_id#11412129, supplier_id#11412130, canonical_id#11412137, is_primary#11412138, entity_name#11412131, city#11412132, state#11412133, country#11412134, threshold#11412139, create_date#11412140], [isnotnull(canonical_id#11412137)]
+- InMemoryRelation [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, canonical_id#11412137, is_primary#11412138, threshold#11412139, create_date#11412140], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#11412129,supplier_id#11412130,entity_name#11412131,city#11412132,state#11412133,country#11412134,supplier_type#11412135,declaring_supplier_id#11412136,canonical_id#11412137,is_primary#11412138,threshold#11412139,create_date#11412140] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(42) AdaptiveSparkPlan
Output [10]: [brand_context_id#11412129, supplier_id#11412130, canonical_id#11412137, is_primary#11412138, entity_name#11412131, city#11412132, state#11412133, country#11412134, threshold#11412139, create_date#11412140]
Arguments: isFinalPlan=false
(43) Filter
Input [4]: [brand_context_id#11440783, canonical_id#11440791, is_primary#11440792, supplier_id#11440784]
Condition : ((((isnotnull(is_primary#11440792) AND is_primary#11440792) AND isnotnull(canonical_id#11440791)) AND isnotnull(brand_context_id#11440783)) AND isnotnull(supplier_id#11440784))
(44) Project
Output [2]: [brand_context_id#11440783, supplier_id#11440784]
Input [4]: [brand_context_id#11440783, canonical_id#11440791, is_primary#11440792, supplier_id#11440784]
(45) BroadcastHashJoin
Left keys [2]: [brand_context_id#11438473, supplier_id#11438474]
Right keys [2]: [brand_context_id#11440783, supplier_id#11440784]
Join type: Inner
Join condition: None
(46) Project
Output [2]: [brand_context_id#11438473, supplier_id#11438474]
Input [4]: [brand_context_id#11438473, supplier_id#11438474, brand_context_id#11440783, supplier_id#11440784]
(47) Exchange
Input [2]: [brand_context_id#11438473, supplier_id#11438474]
Arguments: hashpartitioning(brand_context_id#11438473, supplier_id#11438474, 10), ENSURE_REQUIREMENTS, [plan_id=587864]
(48) Sort
Input [2]: [brand_context_id#11438473, supplier_id#11438474]
Arguments: [brand_context_id#11438473 ASC NULLS FIRST, supplier_id#11438474 ASC NULLS FIRST], false, 0
(49) SortMergeJoin
Left keys [2]: [brand_context_id#11440771, supplier_id#11440772]
Right keys [2]: [brand_context_id#11438473, supplier_id#11438474]
Join type: LeftAnti
Join condition: None
(50) Project
Output [7]: [brand_context_id#11440771 AS u_brand_context_id#11438504, supplier_id#11440772 AS u_supplier_id#11438570, declaring_supplier_id#11440778 AS u_declaring_supplier_id#11438676, supplier_type#11440777 AS u_supplier_type#11438677, threshold#11440781 AS u_threshold#11438680, normalized_name#11437594 AS u_normalized_name#11438681, location_key#11437651 AS u_location_key#11438790]
Input [7]: [brand_context_id#11440771, supplier_id#11440772, supplier_type#11440777, declaring_supplier_id#11440778, threshold#11440781, normalized_name#11437594, location_key#11437651]
(51) BroadcastExchange
Input [7]: [u_brand_context_id#11438504, u_supplier_id#11438570, u_declaring_supplier_id#11438676, u_supplier_type#11438677, u_threshold#11438680, u_normalized_name#11438681, u_location_key#11438790]
Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[6, string, false]),false), [plan_id=587870]
(52) InMemoryTableScan
Output [8]: [brand_context_id#11439375, supplier_id#11439376, canonical_id#11439383, entity_name#11439377, city#11439378, state#11439379, country#11439380, create_date#11439386]
Arguments: [brand_context_id#11439375, supplier_id#11439376, canonical_id#11439383, entity_name#11439377, city#11439378, state#11439379, country#11439380, create_date#11439386], [isnotnull(brand_context_id#11439375)]
(53) InMemoryRelation
Arguments: [brand_context_id#11439375, supplier_id#11439376, canonical_id#11439383, is_primary#11439384, entity_name#11439377, city#11439378, state#11439379, country#11439380, threshold#11439385, create_date#11439386], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@7795a5ad,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Filter isnotnull(canonical_id#11412137)
+- TableCacheQueryStage 0
+- InMemoryTableScan [brand_context_id#11412129, supplier_id#11412130, canonical_id#11412137, is_primary#11412138, entity_name#11412131, city#11412132, state#11412133, country#11412134, threshold#11412139, create_date#11412140], [isnotnull(canonical_id#11412137)]
+- InMemoryRelation [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, canonical_id#11412137, is_primary#11412138, threshold#11412139, create_date#11412140], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#11412129,supplier_id#11412130,entity_name#11412131,city#11412132,state#11412133,country#11412134,supplier_type#11412135,declaring_supplier_id#11412136,canonical_id#11412137,is_primary#11412138,threshold#11412139,create_date#11412140] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Filter isnotnull(canonical_id#11412137)
+- InMemoryTableScan [brand_context_id#11412129, supplier_id#11412130, canonical_id#11412137, is_primary#11412138, entity_name#11412131, city#11412132, state#11412133, country#11412134, threshold#11412139, create_date#11412140], [isnotnull(canonical_id#11412137)]
+- InMemoryRelation [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, canonical_id#11412137, is_primary#11412138, threshold#11412139, create_date#11412140], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#11412129,supplier_id#11412130,entity_name#11412131,city#11412132,state#11412133,country#11412134,supplier_type#11412135,declaring_supplier_id#11412136,canonical_id#11412137,is_primary#11412138,threshold#11412139,create_date#11412140] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(54) AdaptiveSparkPlan
Output [10]: [brand_context_id#11412129, supplier_id#11412130, canonical_id#11412137, is_primary#11412138, entity_name#11412131, city#11412132, state#11412133, country#11412134, threshold#11412139, create_date#11412140]
Arguments: isFinalPlan=false
(55) Filter
Input [8]: [brand_context_id#11439375, supplier_id#11439376, canonical_id#11439383, entity_name#11439377, city#11439378, state#11439379, country#11439380, create_date#11439386]
Condition : isnotnull(brand_context_id#11439375)
(56) BatchEvalPython
Input [8]: [brand_context_id#11439375, supplier_id#11439376, canonical_id#11439383, entity_name#11439377, city#11439378, state#11439379, country#11439380, create_date#11439386]
Arguments: [normalize_name(entity_name#11439377)#11437672, normalize_location(city#11439378)#11437732, normalize_location(state#11439379)#11437750, normalize_location(country#11439380)#11437769], [pythonUDF0#11442424, pythonUDF1#11442425, pythonUDF2#11442426, pythonUDF3#11442427]
(57) Project
Output [6]: [brand_context_id#11439375 AS e_brand_context_id#11438803, supplier_id#11439376 AS e_supplier_id#11438804, canonical_id#11439383 AS e_canonical_id#11438805, pythonUDF0#11442424 AS e_normalized_name#11438850, concat_ws(|, coalesce(pythonUDF1#11442425, ), coalesce(pythonUDF2#11442426, ), coalesce(pythonUDF3#11442427, )) AS e_location_key#11438896, create_date#11439386 AS e_create_date#11438926]
Input [12]: [brand_context_id#11439375, supplier_id#11439376, canonical_id#11439383, entity_name#11439377, city#11439378, state#11439379, country#11439380, create_date#11439386, pythonUDF0#11442424, pythonUDF1#11442425, pythonUDF2#11442426, pythonUDF3#11442427]
(58) BroadcastHashJoin
Left keys [2]: [u_brand_context_id#11438504, u_location_key#11438790]
Right keys [2]: [e_brand_context_id#11438803, e_location_key#11438896]
Join type: Inner
Join condition: None
(59) Project
Output [10]: [u_brand_context_id#11438504, u_supplier_id#11438570, u_declaring_supplier_id#11438676, u_supplier_type#11438677, u_threshold#11438680, u_normalized_name#11438681, e_supplier_id#11438804, e_canonical_id#11438805, e_normalized_name#11438850, e_create_date#11438926]
Input [13]: [u_brand_context_id#11438504, u_supplier_id#11438570, u_declaring_supplier_id#11438676, u_supplier_type#11438677, u_threshold#11438680, u_normalized_name#11438681, u_location_key#11438790, e_brand_context_id#11438803, e_supplier_id#11438804, e_canonical_id#11438805, e_normalized_name#11438850, e_location_key#11438896, e_create_date#11438926]
(60) BatchEvalPython
Input [10]: [u_brand_context_id#11438504, u_supplier_id#11438570, u_declaring_supplier_id#11438676, u_supplier_type#11438677, u_threshold#11438680, u_normalized_name#11438681, e_supplier_id#11438804, e_canonical_id#11438805, e_normalized_name#11438850, e_create_date#11438926]
Arguments: [calculate_similarity(u_normalized_name#11438681, e_normalized_name#11438850)#11440051], [pythonUDF0#11442428]
(61) Filter
Input [11]: [u_brand_context_id#11438504, u_supplier_id#11438570, u_declaring_supplier_id#11438676, u_supplier_type#11438677, u_threshold#11438680, u_normalized_name#11438681, e_supplier_id#11438804, e_canonical_id#11438805, e_normalized_name#11438850, e_create_date#11438926, pythonUDF0#11442428]
Condition : (cast(pythonUDF0#11442428 as double) >= u_threshold#11438680)
(62) Project
Output [10]: [u_brand_context_id#11438504, u_supplier_id#11438570, u_declaring_supplier_id#11438676, u_supplier_type#11438677, u_threshold#11438680, u_normalized_name#11438681, e_supplier_id#11438804, e_canonical_id#11438805, e_normalized_name#11438850, e_create_date#11438926]
Input [11]: [u_brand_context_id#11438504, u_supplier_id#11438570, u_declaring_supplier_id#11438676, u_supplier_type#11438677, u_threshold#11438680, u_normalized_name#11438681, e_supplier_id#11438804, e_canonical_id#11438805, e_normalized_name#11438850, e_create_date#11438926, pythonUDF0#11442428]
(63) BatchEvalPython
Input [10]: [u_brand_context_id#11438504, u_supplier_id#11438570, u_declaring_supplier_id#11438676, u_supplier_type#11438677, u_threshold#11438680, u_normalized_name#11438681, e_supplier_id#11438804, e_canonical_id#11438805, e_normalized_name#11438850, e_create_date#11438926]
Arguments: [calculate_similarity(u_normalized_name#11438681, e_normalized_name#11438850)#11440051], [pythonUDF0#11442429]
(64) Project
Output [9]: [u_brand_context_id#11438504, u_supplier_id#11438570, u_declaring_supplier_id#11438676, u_supplier_type#11438677, u_threshold#11438680, e_supplier_id#11438804, e_canonical_id#11438805, e_create_date#11438926, pythonUDF0#11442429 AS similarity#11440052]
Input [11]: [u_brand_context_id#11438504, u_supplier_id#11438570, u_declaring_supplier_id#11438676, u_supplier_type#11438677, u_threshold#11438680, u_normalized_name#11438681, e_supplier_id#11438804, e_canonical_id#11438805, e_normalized_name#11438850, e_create_date#11438926, pythonUDF0#11442429]
(65) Sort
Input [9]: [u_brand_context_id#11438504, u_supplier_id#11438570, u_declaring_supplier_id#11438676, u_supplier_type#11438677, u_threshold#11438680, e_supplier_id#11438804, e_canonical_id#11438805, e_create_date#11438926, similarity#11440052]
Arguments: [u_brand_context_id#11438504 ASC NULLS FIRST, u_supplier_id#11438570 ASC NULLS FIRST, similarity#11440052 DESC NULLS LAST, e_create_date#11438926 ASC NULLS FIRST, e_supplier_id#11438804 ASC NULLS FIRST], false, 0
(66) WindowGroupLimit
Input [9]: [u_brand_context_id#11438504, u_supplier_id#11438570, u_declaring_supplier_id#11438676, u_supplier_type#11438677, u_threshold#11438680, e_supplier_id#11438804, e_canonical_id#11438805, e_create_date#11438926, similarity#11440052]
Arguments: [u_brand_context_id#11438504, u_supplier_id#11438570], [similarity#11440052 DESC NULLS LAST, e_create_date#11438926 ASC NULLS FIRST, e_supplier_id#11438804 ASC NULLS FIRST], row_number(), 1, Partial
(67) Exchange
Input [9]: [u_brand_context_id#11438504, u_supplier_id#11438570, u_declaring_supplier_id#11438676, u_supplier_type#11438677, u_threshold#11438680, e_supplier_id#11438804, e_canonical_id#11438805, e_create_date#11438926, similarity#11440052]
Arguments: hashpartitioning(u_brand_context_id#11438504, u_supplier_id#11438570, 10), ENSURE_REQUIREMENTS, [plan_id=587882]
(68) Sort
Input [9]: [u_brand_context_id#11438504, u_supplier_id#11438570, u_declaring_supplier_id#11438676, u_supplier_type#11438677, u_threshold#11438680, e_supplier_id#11438804, e_canonical_id#11438805, e_create_date#11438926, similarity#11440052]
Arguments: [u_brand_context_id#11438504 ASC NULLS FIRST, u_supplier_id#11438570 ASC NULLS FIRST, similarity#11440052 DESC NULLS LAST, e_create_date#11438926 ASC NULLS FIRST, e_supplier_id#11438804 ASC NULLS FIRST], false, 0
(69) WindowGroupLimit
Input [9]: [u_brand_context_id#11438504, u_supplier_id#11438570, u_declaring_supplier_id#11438676, u_supplier_type#11438677, u_threshold#11438680, e_supplier_id#11438804, e_canonical_id#11438805, e_create_date#11438926, similarity#11440052]
Arguments: [u_brand_context_id#11438504, u_supplier_id#11438570], [similarity#11440052 DESC NULLS LAST, e_create_date#11438926 ASC NULLS FIRST, e_supplier_id#11438804 ASC NULLS FIRST], row_number(), 1, Final
(70) Window
Input [9]: [u_brand_context_id#11438504, u_supplier_id#11438570, u_declaring_supplier_id#11438676, u_supplier_type#11438677, u_threshold#11438680, e_supplier_id#11438804, e_canonical_id#11438805, e_create_date#11438926, similarity#11440052]
Arguments: [row_number() windowspecdefinition(u_brand_context_id#11438504, u_supplier_id#11438570, similarity#11440052 DESC NULLS LAST, e_create_date#11438926 ASC NULLS FIRST, e_supplier_id#11438804 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank#11440316], [u_brand_context_id#11438504, u_supplier_id#11438570], [similarity#11440052 DESC NULLS LAST, e_create_date#11438926 ASC NULLS FIRST, e_supplier_id#11438804 ASC NULLS FIRST]
(71) Filter
Input [10]: [u_brand_context_id#11438504, u_supplier_id#11438570, u_declaring_supplier_id#11438676, u_supplier_type#11438677, u_threshold#11438680, e_supplier_id#11438804, e_canonical_id#11438805, e_create_date#11438926, similarity#11440052, rank#11440316]
Condition : (rank#11440316 = 1)
(72) Project
Output [10]: [u_brand_context_id#11438504 AS brand_context_id#11440593, u_supplier_id#11438570 AS supplier_id#11440594, u_declaring_supplier_id#11438676 AS declaring_supplier_id#11440595, u_supplier_type#11438677 AS supplier_type#11440596, e_canonical_id#11438805 AS canonical_id#11440617, 0 AS is_primary#11440618, u_threshold#11438680 AS configured_threshold#11440619, cast((similarity#11440052 * 100.0) as double) AS match_percentage#11440976, cast(similarity#11440052 as double) AS confidence_score#11440977, matched_existing AS match_reason#11440622]
Input [10]: [u_brand_context_id#11438504, u_supplier_id#11438570, u_declaring_supplier_id#11438676, u_supplier_type#11438677, u_threshold#11438680, e_supplier_id#11438804, e_canonical_id#11438805, e_create_date#11438926, similarity#11440052, rank#11440316]
(73) InMemoryTableScan
Output [5]: [brand_context_id#11441045, supplier_id#11441046, supplier_type#11441051, declaring_supplier_id#11441052, threshold#11441055]
Arguments: [brand_context_id#11441045, supplier_id#11441046, supplier_type#11441051, declaring_supplier_id#11441052, threshold#11441055]
(74) InMemoryRelation
Arguments: [brand_context_id#11441045, supplier_id#11441046, entity_name#11441047, city#11441048, state#11441049, country#11441050, supplier_type#11441051, declaring_supplier_id#11441052, threshold#11441055, create_date#11441056], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@7795a5ad,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Project [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, threshold#11412139, create_date#11412140]
+- *(1) Filter isnull(canonical_id#11412137)
+- TableCacheQueryStage 0
+- InMemoryTableScan [brand_context_id#11412129, canonical_id#11412137, city#11412132, country#11412134, create_date#11412140, declaring_supplier_id#11412136, entity_name#11412131, state#11412133, supplier_id#11412130, supplier_type#11412135, threshold#11412139], [isnull(canonical_id#11412137)]
+- InMemoryRelation [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, canonical_id#11412137, is_primary#11412138, threshold#11412139, create_date#11412140], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#11412129,supplier_id#11412130,entity_name#11412131,city#11412132,state#11412133,country#11412134,supplier_type#11412135,declaring_supplier_id#11412136,canonical_id#11412137,is_primary#11412138,threshold#11412139,create_date#11412140] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Project [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, threshold#11412139, create_date#11412140]
+- Filter isnull(canonical_id#11412137)
+- InMemoryTableScan [brand_context_id#11412129, canonical_id#11412137, city#11412132, country#11412134, create_date#11412140, declaring_supplier_id#11412136, entity_name#11412131, state#11412133, supplier_id#11412130, supplier_type#11412135, threshold#11412139], [isnull(canonical_id#11412137)]
+- InMemoryRelation [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, canonical_id#11412137, is_primary#11412138, threshold#11412139, create_date#11412140], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#11412129,supplier_id#11412130,entity_name#11412131,city#11412132,state#11412133,country#11412134,supplier_type#11412135,declaring_supplier_id#11412136,canonical_id#11412137,is_primary#11412138,threshold#11412139,create_date#11412140] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(75) AdaptiveSparkPlan
Output [10]: [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, threshold#11412139, create_date#11412140]
Arguments: isFinalPlan=false
(76) Exchange
Input [5]: [brand_context_id#11441045, supplier_id#11441046, supplier_type#11441051, declaring_supplier_id#11441052, threshold#11441055]
Arguments: hashpartitioning(brand_context_id#11441045, supplier_id#11441046, 10), ENSURE_REQUIREMENTS, [plan_id=587893]
(77) Sort
Input [5]: [brand_context_id#11441045, supplier_id#11441046, supplier_type#11441051, declaring_supplier_id#11441052, threshold#11441055]
Arguments: [brand_context_id#11441045 ASC NULLS FIRST, supplier_id#11441046 ASC NULLS FIRST], false, 0
(78) InMemoryTableScan
Output [2]: [brand_context_id#11441057, supplier_id#11441058]
Arguments: [brand_context_id#11441057, supplier_id#11441058], [isnotnull(brand_context_id#11441057), isnotnull(supplier_id#11441058)]
(79) InMemoryRelation
Arguments: [brand_context_id#11441057, supplier_id#11441058, entity_name#11441059, city#11441060, state#11441061, country#11441062, supplier_type#11441063, declaring_supplier_id#11441064, threshold#11441067, create_date#11441068], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@7795a5ad,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Project [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, threshold#11412139, create_date#11412140]
+- *(1) Filter isnull(canonical_id#11412137)
+- TableCacheQueryStage 0
+- InMemoryTableScan [brand_context_id#11412129, canonical_id#11412137, city#11412132, country#11412134, create_date#11412140, declaring_supplier_id#11412136, entity_name#11412131, state#11412133, supplier_id#11412130, supplier_type#11412135, threshold#11412139], [isnull(canonical_id#11412137)]
+- InMemoryRelation [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, canonical_id#11412137, is_primary#11412138, threshold#11412139, create_date#11412140], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#11412129,supplier_id#11412130,entity_name#11412131,city#11412132,state#11412133,country#11412134,supplier_type#11412135,declaring_supplier_id#11412136,canonical_id#11412137,is_primary#11412138,threshold#11412139,create_date#11412140] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Project [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, threshold#11412139, create_date#11412140]
+- Filter isnull(canonical_id#11412137)
+- InMemoryTableScan [brand_context_id#11412129, canonical_id#11412137, city#11412132, country#11412134, create_date#11412140, declaring_supplier_id#11412136, entity_name#11412131, state#11412133, supplier_id#11412130, supplier_type#11412135, threshold#11412139], [isnull(canonical_id#11412137)]
+- InMemoryRelation [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, canonical_id#11412137, is_primary#11412138, threshold#11412139, create_date#11412140], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#11412129,supplier_id#11412130,entity_name#11412131,city#11412132,state#11412133,country#11412134,supplier_type#11412135,declaring_supplier_id#11412136,canonical_id#11412137,is_primary#11412138,threshold#11412139,create_date#11412140] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(80) AdaptiveSparkPlan
Output [10]: [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, threshold#11412139, create_date#11412140]
Arguments: isFinalPlan=false
(81) Filter
Input [2]: [brand_context_id#11441057, supplier_id#11441058]
Condition : (isnotnull(brand_context_id#11441057) AND isnotnull(supplier_id#11441058))
(82) BroadcastExchange
Input [2]: [brand_context_id#11441057, supplier_id#11441058]
Arguments: HashedRelationBroadcastMode(List(input[0, string, false], input[1, string, false]),false), [plan_id=587888]
(83) InMemoryTableScan
Output [4]: [brand_context_id#11441069, canonical_id#11441077, is_primary#11441078, supplier_id#11441070]
Arguments: [brand_context_id#11441069, canonical_id#11441077, is_primary#11441078, supplier_id#11441070], [isnotnull(is_primary#11441078), is_primary#11441078, isnotnull(canonical_id#11441077), isnotnull(brand_context_id#11441069), isnotnull(supplier_id#11441070)]
(84) InMemoryRelation
Arguments: [brand_context_id#11441069, supplier_id#11441070, canonical_id#11441077, is_primary#11441078, entity_name#11441071, city#11441072, state#11441073, country#11441074, threshold#11441079, create_date#11441080], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@7795a5ad,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Filter isnotnull(canonical_id#11412137)
+- TableCacheQueryStage 0
+- InMemoryTableScan [brand_context_id#11412129, supplier_id#11412130, canonical_id#11412137, is_primary#11412138, entity_name#11412131, city#11412132, state#11412133, country#11412134, threshold#11412139, create_date#11412140], [isnotnull(canonical_id#11412137)]
+- InMemoryRelation [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, canonical_id#11412137, is_primary#11412138, threshold#11412139, create_date#11412140], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#11412129,supplier_id#11412130,entity_name#11412131,city#11412132,state#11412133,country#11412134,supplier_type#11412135,declaring_supplier_id#11412136,canonical_id#11412137,is_primary#11412138,threshold#11412139,create_date#11412140] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Filter isnotnull(canonical_id#11412137)
+- InMemoryTableScan [brand_context_id#11412129, supplier_id#11412130, canonical_id#11412137, is_primary#11412138, entity_name#11412131, city#11412132, state#11412133, country#11412134, threshold#11412139, create_date#11412140], [isnotnull(canonical_id#11412137)]
+- InMemoryRelation [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, canonical_id#11412137, is_primary#11412138, threshold#11412139, create_date#11412140], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#11412129,supplier_id#11412130,entity_name#11412131,city#11412132,state#11412133,country#11412134,supplier_type#11412135,declaring_supplier_id#11412136,canonical_id#11412137,is_primary#11412138,threshold#11412139,create_date#11412140] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(85) AdaptiveSparkPlan
Output [10]: [brand_context_id#11412129, supplier_id#11412130, canonical_id#11412137, is_primary#11412138, entity_name#11412131, city#11412132, state#11412133, country#11412134, threshold#11412139, create_date#11412140]
Arguments: isFinalPlan=false
(86) Filter
Input [4]: [brand_context_id#11441069, canonical_id#11441077, is_primary#11441078, supplier_id#11441070]
Condition : ((((isnotnull(is_primary#11441078) AND is_primary#11441078) AND isnotnull(canonical_id#11441077)) AND isnotnull(brand_context_id#11441069)) AND isnotnull(supplier_id#11441070))
(87) Project
Output [2]: [brand_context_id#11441069, supplier_id#11441070]
Input [4]: [brand_context_id#11441069, canonical_id#11441077, is_primary#11441078, supplier_id#11441070]
(88) BroadcastHashJoin
Left keys [2]: [brand_context_id#11441057, supplier_id#11441058]
Right keys [2]: [brand_context_id#11441069, supplier_id#11441070]
Join type: Inner
Join condition: None
(89) Project
Output [2]: [brand_context_id#11441057, supplier_id#11441058]
Input [4]: [brand_context_id#11441057, supplier_id#11441058, brand_context_id#11441069, supplier_id#11441070]
(90) Exchange
Input [2]: [brand_context_id#11441057, supplier_id#11441058]
Arguments: hashpartitioning(brand_context_id#11441057, supplier_id#11441058, 10), ENSURE_REQUIREMENTS, [plan_id=587894]
(91) Sort
Input [2]: [brand_context_id#11441057, supplier_id#11441058]
Arguments: [brand_context_id#11441057 ASC NULLS FIRST, supplier_id#11441058 ASC NULLS FIRST], false, 0
(92) SortMergeJoin
Left keys [2]: [brand_context_id#11441045, supplier_id#11441046]
Right keys [2]: [brand_context_id#11441057, supplier_id#11441058]
Join type: LeftAnti
Join condition: None
(93) InMemoryTableScan
Output [7]: [brand_context_id#11440635, supplier_id#11440636, entity_name#11440637, city#11440638, state#11440639, country#11440640, threshold#11440645]
Arguments: [brand_context_id#11440635, supplier_id#11440636, entity_name#11440637, city#11440638, state#11440639, country#11440640, threshold#11440645], [isnotnull(threshold#11440645), isnotnull(brand_context_id#11440635), isnotnull(supplier_id#11440636)]
(94) InMemoryRelation
Arguments: [brand_context_id#11440635, supplier_id#11440636, entity_name#11440637, city#11440638, state#11440639, country#11440640, supplier_type#11440641, declaring_supplier_id#11440642, threshold#11440645, create_date#11440646], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@7795a5ad,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Project [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, threshold#11412139, create_date#11412140]
+- *(1) Filter isnull(canonical_id#11412137)
+- TableCacheQueryStage 0
+- InMemoryTableScan [brand_context_id#11412129, canonical_id#11412137, city#11412132, country#11412134, create_date#11412140, declaring_supplier_id#11412136, entity_name#11412131, state#11412133, supplier_id#11412130, supplier_type#11412135, threshold#11412139], [isnull(canonical_id#11412137)]
+- InMemoryRelation [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, canonical_id#11412137, is_primary#11412138, threshold#11412139, create_date#11412140], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#11412129,supplier_id#11412130,entity_name#11412131,city#11412132,state#11412133,country#11412134,supplier_type#11412135,declaring_supplier_id#11412136,canonical_id#11412137,is_primary#11412138,threshold#11412139,create_date#11412140] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Project [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, threshold#11412139, create_date#11412140]
+- Filter isnull(canonical_id#11412137)
+- InMemoryTableScan [brand_context_id#11412129, canonical_id#11412137, city#11412132, country#11412134, create_date#11412140, declaring_supplier_id#11412136, entity_name#11412131, state#11412133, supplier_id#11412130, supplier_type#11412135, threshold#11412139], [isnull(canonical_id#11412137)]
+- InMemoryRelation [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, canonical_id#11412137, is_primary#11412138, threshold#11412139, create_date#11412140], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#11412129,supplier_id#11412130,entity_name#11412131,city#11412132,state#11412133,country#11412134,supplier_type#11412135,declaring_supplier_id#11412136,canonical_id#11412137,is_primary#11412138,threshold#11412139,create_date#11412140] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(95) AdaptiveSparkPlan
Output [10]: [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, threshold#11412139, create_date#11412140]
Arguments: isFinalPlan=false
(96) Filter
Input [7]: [brand_context_id#11440635, supplier_id#11440636, entity_name#11440637, city#11440638, state#11440639, country#11440640, threshold#11440645]
Condition : ((isnotnull(threshold#11440645) AND isnotnull(brand_context_id#11440635)) AND isnotnull(supplier_id#11440636))
(97) BatchEvalPython
Input [7]: [brand_context_id#11440635, supplier_id#11440636, entity_name#11440637, city#11440638, state#11440639, country#11440640, threshold#11440645]
Arguments: [normalize_location(city#11440638)#11437606, normalize_location(state#11440639)#11437620, normalize_location(country#11440640)#11437635], [pythonUDF0#11442430, pythonUDF1#11442431, pythonUDF2#11442432]
(98) Filter
Input [10]: [brand_context_id#11440635, supplier_id#11440636, entity_name#11440637, city#11440638, state#11440639, country#11440640, threshold#11440645, pythonUDF0#11442430, pythonUDF1#11442431, pythonUDF2#11442432]
Condition : ((isnotnull(pythonUDF0#11442430) AND isnotnull(pythonUDF1#11442431)) AND isnotnull(pythonUDF2#11442432))
(99) Project
Output [7]: [brand_context_id#11440635, supplier_id#11440636, entity_name#11440637, city#11440638, state#11440639, country#11440640, threshold#11440645]
Input [10]: [brand_context_id#11440635, supplier_id#11440636, entity_name#11440637, city#11440638, state#11440639, country#11440640, threshold#11440645, pythonUDF0#11442430, pythonUDF1#11442431, pythonUDF2#11442432]
(100) BatchEvalPython
Input [7]: [brand_context_id#11440635, supplier_id#11440636, entity_name#11440637, city#11440638, state#11440639, country#11440640, threshold#11440645]
Arguments: [normalize_name(entity_name#11440637)#11437593, normalize_location(city#11440638)#11437606, normalize_location(state#11440639)#11437620, normalize_location(country#11440640)#11437635], [pythonUDF0#11442433, pythonUDF1#11442434, pythonUDF2#11442435, pythonUDF3#11442436]
(101) Project
Output [5]: [brand_context_id#11440635, supplier_id#11440636, threshold#11440645, pythonUDF0#11442433 AS normalized_name#11437594, concat_ws(|, coalesce(pythonUDF1#11442434, ), coalesce(pythonUDF2#11442435, ), coalesce(pythonUDF3#11442436, )) AS location_key#11437651]
Input [11]: [brand_context_id#11440635, supplier_id#11440636, entity_name#11440637, city#11440638, state#11440639, country#11440640, threshold#11440645, pythonUDF0#11442433, pythonUDF1#11442434, pythonUDF2#11442435, pythonUDF3#11442436]
(102) Exchange
Input [5]: [brand_context_id#11440635, supplier_id#11440636, threshold#11440645, normalized_name#11437594, location_key#11437651]
Arguments: hashpartitioning(brand_context_id#11440635, supplier_id#11440636, 10), ENSURE_REQUIREMENTS, [plan_id=587903]
(103) Sort
Input [5]: [brand_context_id#11440635, supplier_id#11440636, threshold#11440645, normalized_name#11437594, location_key#11437651]
Arguments: [brand_context_id#11440635 ASC NULLS FIRST, supplier_id#11440636 ASC NULLS FIRST], false, 0
(104) InMemoryTableScan
Output [2]: [brand_context_id#11440647, supplier_id#11440648]
Arguments: [brand_context_id#11440647, supplier_id#11440648], [isnotnull(brand_context_id#11440647), isnotnull(supplier_id#11440648)]
(105) InMemoryRelation
Arguments: [brand_context_id#11440647, supplier_id#11440648, entity_name#11440649, city#11440650, state#11440651, country#11440652, supplier_type#11440653, declaring_supplier_id#11440654, threshold#11440657, create_date#11440658], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@7795a5ad,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Project [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, threshold#11412139, create_date#11412140]
+- *(1) Filter isnull(canonical_id#11412137)
+- TableCacheQueryStage 0
+- InMemoryTableScan [brand_context_id#11412129, canonical_id#11412137, city#11412132, country#11412134, create_date#11412140, declaring_supplier_id#11412136, entity_name#11412131, state#11412133, supplier_id#11412130, supplier_type#11412135, threshold#11412139], [isnull(canonical_id#11412137)]
+- InMemoryRelation [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, canonical_id#11412137, is_primary#11412138, threshold#11412139, create_date#11412140], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#11412129,supplier_id#11412130,entity_name#11412131,city#11412132,state#11412133,country#11412134,supplier_type#11412135,declaring_supplier_id#11412136,canonical_id#11412137,is_primary#11412138,threshold#11412139,create_date#11412140] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Project [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, threshold#11412139, create_date#11412140]
+- Filter isnull(canonical_id#11412137)
+- InMemoryTableScan [brand_context_id#11412129, canonical_id#11412137, city#11412132, country#11412134, create_date#11412140, declaring_supplier_id#11412136, entity_name#11412131, state#11412133, supplier_id#11412130, supplier_type#11412135, threshold#11412139], [isnull(canonical_id#11412137)]
+- InMemoryRelation [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, canonical_id#11412137, is_primary#11412138, threshold#11412139, create_date#11412140], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#11412129,supplier_id#11412130,entity_name#11412131,city#11412132,state#11412133,country#11412134,supplier_type#11412135,declaring_supplier_id#11412136,canonical_id#11412137,is_primary#11412138,threshold#11412139,create_date#11412140] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(106) AdaptiveSparkPlan
Output [10]: [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, threshold#11412139, create_date#11412140]
Arguments: isFinalPlan=false
(107) Filter
Input [2]: [brand_context_id#11440647, supplier_id#11440648]
Condition : (isnotnull(brand_context_id#11440647) AND isnotnull(supplier_id#11440648))
(108) BroadcastExchange
Input [2]: [brand_context_id#11440647, supplier_id#11440648]
Arguments: HashedRelationBroadcastMode(List(input[0, string, false], input[1, string, false]),false), [plan_id=587898]
(109) InMemoryTableScan
Output [4]: [brand_context_id#11440659, canonical_id#11440667, is_primary#11440668, supplier_id#11440660]
Arguments: [brand_context_id#11440659, canonical_id#11440667, is_primary#11440668, supplier_id#11440660], [isnotnull(is_primary#11440668), is_primary#11440668, isnotnull(canonical_id#11440667), isnotnull(brand_context_id#11440659), isnotnull(supplier_id#11440660)]
(110) InMemoryRelation
Arguments: [brand_context_id#11440659, supplier_id#11440660, canonical_id#11440667, is_primary#11440668, entity_name#11440661, city#11440662, state#11440663, country#11440664, threshold#11440669, create_date#11440670], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@7795a5ad,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Filter isnotnull(canonical_id#11412137)
+- TableCacheQueryStage 0
+- InMemoryTableScan [brand_context_id#11412129, supplier_id#11412130, canonical_id#11412137, is_primary#11412138, entity_name#11412131, city#11412132, state#11412133, country#11412134, threshold#11412139, create_date#11412140], [isnotnull(canonical_id#11412137)]
+- InMemoryRelation [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, canonical_id#11412137, is_primary#11412138, threshold#11412139, create_date#11412140], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#11412129,supplier_id#11412130,entity_name#11412131,city#11412132,state#11412133,country#11412134,supplier_type#11412135,declaring_supplier_id#11412136,canonical_id#11412137,is_primary#11412138,threshold#11412139,create_date#11412140] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Filter isnotnull(canonical_id#11412137)
+- InMemoryTableScan [brand_context_id#11412129, supplier_id#11412130, canonical_id#11412137, is_primary#11412138, entity_name#11412131, city#11412132, state#11412133, country#11412134, threshold#11412139, create_date#11412140], [isnotnull(canonical_id#11412137)]
+- InMemoryRelation [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, canonical_id#11412137, is_primary#11412138, threshold#11412139, create_date#11412140], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#11412129,supplier_id#11412130,entity_name#11412131,city#11412132,state#11412133,country#11412134,supplier_type#11412135,declaring_supplier_id#11412136,canonical_id#11412137,is_primary#11412138,threshold#11412139,create_date#11412140] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(111) AdaptiveSparkPlan
Output [10]: [brand_context_id#11412129, supplier_id#11412130, canonical_id#11412137, is_primary#11412138, entity_name#11412131, city#11412132, state#11412133, country#11412134, threshold#11412139, create_date#11412140]
Arguments: isFinalPlan=false
(112) Filter
Input [4]: [brand_context_id#11440659, canonical_id#11440667, is_primary#11440668, supplier_id#11440660]
Condition : ((((isnotnull(is_primary#11440668) AND is_primary#11440668) AND isnotnull(canonical_id#11440667)) AND isnotnull(brand_context_id#11440659)) AND isnotnull(supplier_id#11440660))
(113) Project
Output [2]: [brand_context_id#11440659, supplier_id#11440660]
Input [4]: [brand_context_id#11440659, canonical_id#11440667, is_primary#11440668, supplier_id#11440660]
(114) BroadcastHashJoin
Left keys [2]: [brand_context_id#11440647, supplier_id#11440648]
Right keys [2]: [brand_context_id#11440659, supplier_id#11440660]
Join type: Inner
Join condition: None
(115) Project
Output [2]: [brand_context_id#11440647, supplier_id#11440648]
Input [4]: [brand_context_id#11440647, supplier_id#11440648, brand_context_id#11440659, supplier_id#11440660]
(116) Exchange
Input [2]: [brand_context_id#11440647, supplier_id#11440648]
Arguments: hashpartitioning(brand_context_id#11440647, supplier_id#11440648, 10), ENSURE_REQUIREMENTS, [plan_id=587904]
(117) Sort
Input [2]: [brand_context_id#11440647, supplier_id#11440648]
Arguments: [brand_context_id#11440647 ASC NULLS FIRST, supplier_id#11440648 ASC NULLS FIRST], false, 0
(118) SortMergeJoin
Left keys [2]: [brand_context_id#11440635, supplier_id#11440636]
Right keys [2]: [brand_context_id#11440647, supplier_id#11440648]
Join type: LeftAnti
Join condition: None
(119) Project
Output [5]: [brand_context_id#11440635 AS u_brand_context_id#11438504, supplier_id#11440636 AS u_supplier_id#11438570, threshold#11440645 AS u_threshold#11438680, normalized_name#11437594 AS u_normalized_name#11438681, location_key#11437651 AS u_location_key#11438790]
Input [5]: [brand_context_id#11440635, supplier_id#11440636, threshold#11440645, normalized_name#11437594, location_key#11437651]
(120) BroadcastExchange
Input [5]: [u_brand_context_id#11438504, u_supplier_id#11438570, u_threshold#11438680, u_normalized_name#11438681, u_location_key#11438790]
Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[4, string, false]),false), [plan_id=587910]
(121) InMemoryTableScan
Output [7]: [brand_context_id#11441081, supplier_id#11441082, entity_name#11441083, city#11441084, state#11441085, country#11441086, create_date#11441092]
Arguments: [brand_context_id#11441081, supplier_id#11441082, entity_name#11441083, city#11441084, state#11441085, country#11441086, create_date#11441092], [isnotnull(brand_context_id#11441081)]
(122) InMemoryRelation
Arguments: [brand_context_id#11441081, supplier_id#11441082, canonical_id#11441089, is_primary#11441090, entity_name#11441083, city#11441084, state#11441085, country#11441086, threshold#11441091, create_date#11441092], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@7795a5ad,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Filter isnotnull(canonical_id#11412137)
+- TableCacheQueryStage 0
+- InMemoryTableScan [brand_context_id#11412129, supplier_id#11412130, canonical_id#11412137, is_primary#11412138, entity_name#11412131, city#11412132, state#11412133, country#11412134, threshold#11412139, create_date#11412140], [isnotnull(canonical_id#11412137)]
+- InMemoryRelation [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, canonical_id#11412137, is_primary#11412138, threshold#11412139, create_date#11412140], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#11412129,supplier_id#11412130,entity_name#11412131,city#11412132,state#11412133,country#11412134,supplier_type#11412135,declaring_supplier_id#11412136,canonical_id#11412137,is_primary#11412138,threshold#11412139,create_date#11412140] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Filter isnotnull(canonical_id#11412137)
+- InMemoryTableScan [brand_context_id#11412129, supplier_id#11412130, canonical_id#11412137, is_primary#11412138, entity_name#11412131, city#11412132, state#11412133, country#11412134, threshold#11412139, create_date#11412140], [isnotnull(canonical_id#11412137)]
+- InMemoryRelation [brand_context_id#11412129, supplier_id#11412130, entity_name#11412131, city#11412132, state#11412133, country#11412134, supplier_type#11412135, declaring_supplier_id#11412136, canonical_id#11412137, is_primary#11412138, threshold#11412139, create_date#11412140], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#11412129,supplier_id#11412130,entity_name#11412131,city#11412132,state#11412133,country#11412134,supplier_type#11412135,declaring_supplier_id#11412136,canonical_id#11412137,is_primary#11412138,threshold#11412139,create_date#11412140] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(123) AdaptiveSparkPlan
Output [10]: [brand_context_id#11412129, supplier_id#11412130, canonical_id#11412137, is_primary#11412138, entity_name#11412131, city#11412132, state#11412133, country#11412134, threshold#11412139, create_date#11412140]
Arguments: isFinalPlan=false
(124) Filter
Input [7]: [brand_context_id#11441081, supplier_id#11441082, entity_name#11441083, city#11441084, state#11441085, country#11441086, create_date#11441092]
Condition : isnotnull(brand_context_id#11441081)
(125) BatchEvalPython
Input [7]: [brand_context_id#11441081, supplier_id#11441082, entity_name#11441083, city#11441084, state#11441085, country#11441086, create_date#11441092]
Arguments: [normalize_name(entity_name#11441083)#11437672, normalize_location(city#11441084)#11437732, normalize_location(state#11441085)#11437750, normalize_location(country#11441086)#11437769], [pythonUDF0#11442437, pythonUDF1#11442438, pythonUDF2#11442439, pythonUDF3#11442440]
(126) Project
Output [5]: [brand_context_id#11441081 AS e_brand_context_id#11438803, supplier_id#11441082 AS e_supplier_id#11438804, pythonUDF0#11442437 AS e_normalized_name#11438850, concat_ws(|, coalesce(pythonUDF1#11442438, ), coalesce(pythonUDF2#11442439, ), coalesce(pythonUDF3#11442440, )) AS e_location_key#11438896, create_date#11441092 AS e_create_date#11438926]
Input [11]: [brand_context_id#11441081, supplier_id#11441082, entity_name#11441083, city#11441084, state#11441085, country#11441086, create_date#11441092, pythonUDF0#11442437, pythonUDF1#11442438, pythonUDF2#11442439, pythonUDF3#11442440]
(127) BroadcastHashJoin
Left keys [2]: [u_brand_context_id#11438504, u_location_key#11438790]
Right keys [2]: [e_brand_context_id#11438803, e_location_key#11438896]
Join type: Inner
Join condition: None
(128) Project
Output [7]: [u_brand_context_id#11438504, u_supplier_id#11438570, u_threshold#11438680, u_normalized_name#11438681, e_supplier_id#11438804, e_normalized_name#11438850, e_create_date#11438926]
Input [10]: [u_brand_context_id#11438504, u_supplier_id#11438570, u_threshold#11438680, u_normalized_name#11438681, u_location_key#11438790, e_brand_context_id#11438803, e_supplier_id#11438804, e_normalized_name#11438850, e_location_key#11438896, e_create_date#11438926]
(129) BatchEvalPython
Input [7]: [u_brand_context_id#11438504, u_supplier_id#11438570, u_threshold#11438680, u_normalized_name#11438681, e_supplier_id#11438804, e_normalized_name#11438850, e_create_date#11438926]
Arguments: [calculate_similarity(u_normalized_name#11438681, e_normalized_name#11438850)#11440051], [pythonUDF0#11442441]
(130) Filter
Input [8]: [u_brand_context_id#11438504, u_supplier_id#11438570, u_threshold#11438680, u_normalized_name#11438681, e_supplier_id#11438804, e_normalized_name#11438850, e_create_date#11438926, pythonUDF0#11442441]
Condition : (cast(pythonUDF0#11442441 as double) >= u_threshold#11438680)
(131) Project
Output [6]: [u_brand_context_id#11438504, u_supplier_id#11438570, u_normalized_name#11438681, e_supplier_id#11438804, e_normalized_name#11438850, e_create_date#11438926]
Input [8]: [u_brand_context_id#11438504, u_supplier_id#11438570, u_threshold#11438680, u_normalized_name#11438681, e_supplier_id#11438804, e_normalized_name#11438850, e_create_date#11438926, pythonUDF0#11442441]
(132) BatchEvalPython
Input [6]: [u_brand_context_id#11438504, u_supplier_id#11438570, u_normalized_name#11438681, e_supplier_id#11438804, e_normalized_name#11438850, e_create_date#11438926]
Arguments: [calculate_similarity(u_normalized_name#11438681, e_normalized_name#11438850)#11440051], [pythonUDF0#11442442]
(133) Project
Output [5]: [u_brand_context_id#11438504, u_supplier_id#11438570, e_supplier_id#11438804, e_create_date#11438926, pythonUDF0#11442442 AS similarity#11440052]
Input [7]: [u_brand_context_id#11438504, u_supplier_id#11438570, u_normalized_name#11438681, e_supplier_id#11438804, e_normalized_name#11438850, e_create_date#11438926, pythonUDF0#11442442]
(134) Sort
Input [5]: [u_brand_context_id#11438504, u_supplier_id#11438570, e_supplier_id#11438804, e_create_date#11438926, similarity#11440052]
Arguments: [u_brand_context_id#11438504 ASC NULLS FIRST, u_supplier_id#11438570 ASC NULLS FIRST, similarity#11440052 DESC NULLS LAST, e_create_date#11438926 ASC NULLS FIRST, e_supplier_id#11438804 ASC NULLS FIRST], false, 0
(135) WindowGroupLimit
Input [5]: [u_brand_context_id#11438504, u_supplier_id#11438570, e_supplier_id#11438804, e_create_date#11438926, similarity#11440052]
Arguments: [u_brand_context_id#11438504, u_supplier_id#11438570], [similarity#11440052 DESC NULLS LAST, e_create_date#11438926 ASC NULLS FIRST, e_supplier_id#11438804 ASC NULLS FIRST], row_number(), 1, Partial
(136) Exchange
Input [5]: [u_brand_context_id#11438504, u_supplier_id#11438570, e_supplier_id#11438804, e_create_date#11438926, similarity#11440052]
Arguments: hashpartitioning(u_brand_context_id#11438504, u_supplier_id#11438570, 10), ENSURE_REQUIREMENTS, [plan_id=587922]
(137) Sort
Input [5]: [u_brand_context_id#11438504, u_supplier_id#11438570, e_supplier_id#11438804, e_create_date#11438926, similarity#11440052]
Arguments: [u_brand_context_id#11438504 ASC NULLS FIRST, u_supplier_id#11438570 ASC NULLS FIRST, similarity#11440052 DESC NULLS LAST, e_create_date#11438926 ASC NULLS FIRST, e_supplier_id#11438804 ASC NULLS FIRST], false, 0
(138) WindowGroupLimit
Input [5]: [u_brand_context_id#11438504, u_supplier_id#11438570, e_supplier_id#11438804, e_create_date#11438926, similarity#11440052]
Arguments: [u_brand_context_id#11438504, u_supplier_id#11438570], [similarity#11440052 DESC NULLS LAST, e_create_date#11438926 ASC NULLS FIRST, e_supplier_id#11438804 ASC NULLS FIRST], row_number(), 1, Final
(139) Window
Input [5]: [u_brand_context_id#11438504, u_supplier_id#11438570, e_supplier_id#11438804, e_create_date#11438926, similarity#11440052]
Arguments: [row_number() windowspecdefinition(u_brand_context_id#11438504, u_supplier_id#11438570, similarity#11440052 DESC NULLS LAST, e_create_date#11438926 ASC NULLS FIRST, e_supplier_id#11438804 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank#11440316], [u_brand_context_id#11438504, u_supplier_id#11438570], [similarity#11440052 DESC NULLS LAST, e_create_date#11438926 ASC NULLS FIRST, e_supplier_id#11438804 ASC NULLS FIRST]
(140) Filter
Input [6]: [u_brand_context_id#11438504, u_supplier_id#11438570, e_supplier_id#11438804, e_create_date#11438926, similarity#11440052, rank#11440316]
Condition : (rank#11440316 = 1)
(141) Project
Output [2]: [u_brand_context_id#11438504 AS brand_context_id#11440593, u_supplier_id#11438570 AS supplier_id#11440594]
Input [6]: [u_brand_context_id#11438504, u_supplier_id#11438570, e_supplier_id#11438804, e_create_date#11438926, similarity#11440052, rank#11440316]
(142) SortMergeJoin
Left keys [2]: [brand_context_id#11441045, supplier_id#11441046]
Right keys [2]: [brand_context_id#11440593, supplier_id#11440594]
Join type: LeftAnti
Join condition: None
(143) Project
Output [10]: [brand_context_id#11441045, supplier_id#11441046, declaring_supplier_id#11441052, supplier_type#11441051, supplier_id#11441046 AS canonical_id#11440739, 1 AS is_primary#11440756, threshold#11441055 AS configured_threshold#11440757, 0.0 AS match_percentage#11440758, 0.0 AS confidence_score#11440759, new_canonical AS match_reason#11440760]
Input [5]: [brand_context_id#11441045, supplier_id#11441046, supplier_type#11441051, declaring_supplier_id#11441052, threshold#11441055]
(144) Union
(145) AdaptiveSparkPlan
Output [10]: [brand_context_id#11412129, supplier_id#11412130, declaring_supplier_id#11412136, supplier_type#11412135, canonical_id#11438340, is_primary#11438410, configured_threshold#11438411, match_percentage#11438412, confidence_score#11438415, match_reason#11438416]
Arguments: isFinalPlan=false
(146) TableCacheQueryStage
Output [1]: [match_reason#11438416]
Arguments: 0
(147) HashAggregate [codegen id : 1]
Input [1]: [match_reason#11438416]
Keys [1]: [match_reason#11438416]
Functions [1]: [partial_count(1)]
Aggregate Attributes [1]: [count#11456681L]
Results [2]: [match_reason#11438416, count#11456682L]
(148) Exchange
Input [2]: [match_reason#11438416, count#11456682L]
Arguments: hashpartitioning(match_reason#11438416, 10), ENSURE_REQUIREMENTS, [plan_id=587956]
(149) ShuffleQueryStage
Output [2]: [match_reason#11438416, count#11456682L]
Arguments: 1
(150) AQEShuffleRead
Input [2]: [match_reason#11438416, count#11456682L]
Arguments: coalesced
(151) HashAggregate [codegen id : 2]
Input [2]: [match_reason#11438416, count#11456682L]
Keys [1]: [match_reason#11438416]
Functions [1]: [count(1)]
Aggregate Attributes [1]: [count(1)#11456527L]
Results [2]: [match_reason#11438416, count(1)#11456527L AS count#11456528L]
(152) HashAggregate
Input [1]: [match_reason#11438416]
Keys [1]: [match_reason#11438416]
Functions [1]: [partial_count(1)]
Aggregate Attributes [1]: [count#11456681L]
Results [2]: [match_reason#11438416, count#11456682L]
(153) Exchange
Input [2]: [match_reason#11438416, count#11456682L]
Arguments: hashpartitioning(match_reason#11438416, 10), ENSURE_REQUIREMENTS, [plan_id=587852]
(154) HashAggregate
Input [2]: [match_reason#11438416, count#11456682L]
Keys [1]: [match_reason#11438416]
Functions [1]: [count(1)]
Aggregate Attributes [1]: [count(1)#11456527L]
Results [2]: [match_reason#11438416, count(1)#11456527L AS count#11456528L]
(155) AdaptiveSparkPlan
Output [2]: [match_reason#11438416, count#11456528L]
Arguments: isFinalPlan=true