== Physical Plan ==
AdaptiveSparkPlan (155)
+- == Final Plan ==
* HashAggregate (151)
+- AQEShuffleRead (150)
+- ShuffleQueryStage (149), Statistics(sizeInBytes=120.0 B, rowCount=3)
+- Exchange (148)
+- * HashAggregate (147)
+- TableCacheQueryStage (146), Statistics(sizeInBytes=543.0 B, rowCount=3)
+- InMemoryTableScan (1)
+- InMemoryRelation (2)
+- AdaptiveSparkPlan (145)
+- Union (144)
:- Project (23)
: +- BroadcastHashJoin Inner BuildLeft (22)
: :- BroadcastExchange (12)
: : +- Filter (11)
: : +- InMemoryTableScan (3)
: : +- InMemoryRelation (4)
: : +- AdaptiveSparkPlan (10)
: : +- Project (9)
: : +- Filter (8)
: : +- InMemoryTableScan (5)
: : +- InMemoryRelation (6)
: : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: +- Project (21)
: +- Filter (20)
: +- InMemoryTableScan (13)
: +- InMemoryRelation (14)
: +- AdaptiveSparkPlan (19)
: +- Filter (18)
: +- InMemoryTableScan (15)
: +- InMemoryRelation (16)
: +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (17)
:- Project (72)
: +- Filter (71)
: +- Window (70)
: +- WindowGroupLimit (69)
: +- Sort (68)
: +- Exchange (67)
: +- WindowGroupLimit (66)
: +- Sort (65)
: +- Project (64)
: +- BatchEvalPython (63)
: +- Project (62)
: +- Filter (61)
: +- BatchEvalPython (60)
: +- Project (59)
: +- BroadcastHashJoin Inner BuildLeft (58)
: :- BroadcastExchange (51)
: : +- Project (50)
: : +- SortMergeJoin LeftAnti (49)
: : :- Sort (34)
: : : +- Exchange (33)
: : : +- Project (32)
: : : +- BatchEvalPython (31)
: : : +- Project (30)
: : : +- Filter (29)
: : : +- BatchEvalPython (28)
: : : +- Filter (27)
: : : +- InMemoryTableScan (24)
: : : +- InMemoryRelation (25)
: : : +- AdaptiveSparkPlan (26)
: : : +- Project (9)
: : : +- Filter (8)
: : : +- InMemoryTableScan (5)
: : : +- InMemoryRelation (6)
: : : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: : +- Sort (48)
: : +- Exchange (47)
: : +- Project (46)
: : +- BroadcastHashJoin Inner BuildLeft (45)
: : :- BroadcastExchange (39)
: : : +- Filter (38)
: : : +- InMemoryTableScan (35)
: : : +- InMemoryRelation (36)
: : : +- AdaptiveSparkPlan (37)
: : : +- Project (9)
: : : +- Filter (8)
: : : +- InMemoryTableScan (5)
: : : +- InMemoryRelation (6)
: : : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: : +- Project (44)
: : +- Filter (43)
: : +- InMemoryTableScan (40)
: : +- InMemoryRelation (41)
: : +- AdaptiveSparkPlan (42)
: : +- Filter (18)
: : +- InMemoryTableScan (15)
: : +- InMemoryRelation (16)
: : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (17)
: +- Project (57)
: +- BatchEvalPython (56)
: +- Filter (55)
: +- InMemoryTableScan (52)
: +- InMemoryRelation (53)
: +- AdaptiveSparkPlan (54)
: +- Filter (18)
: +- InMemoryTableScan (15)
: +- InMemoryRelation (16)
: +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (17)
+- Project (143)
+- SortMergeJoin LeftAnti (142)
:- SortMergeJoin LeftAnti (92)
: :- Sort (77)
: : +- Exchange (76)
: : +- InMemoryTableScan (73)
: : +- InMemoryRelation (74)
: : +- AdaptiveSparkPlan (75)
: : +- Project (9)
: : +- Filter (8)
: : +- InMemoryTableScan (5)
: : +- InMemoryRelation (6)
: : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: +- Sort (91)
: +- Exchange (90)
: +- Project (89)
: +- BroadcastHashJoin Inner BuildLeft (88)
: :- BroadcastExchange (82)
: : +- Filter (81)
: : +- InMemoryTableScan (78)
: : +- InMemoryRelation (79)
: : +- AdaptiveSparkPlan (80)
: : +- Project (9)
: : +- Filter (8)
: : +- InMemoryTableScan (5)
: : +- InMemoryRelation (6)
: : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: +- Project (87)
: +- Filter (86)
: +- InMemoryTableScan (83)
: +- InMemoryRelation (84)
: +- AdaptiveSparkPlan (85)
: +- Filter (18)
: +- InMemoryTableScan (15)
: +- InMemoryRelation (16)
: +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (17)
+- Project (141)
+- Filter (140)
+- Window (139)
+- WindowGroupLimit (138)
+- Sort (137)
+- Exchange (136)
+- WindowGroupLimit (135)
+- Sort (134)
+- Project (133)
+- BatchEvalPython (132)
+- Project (131)
+- Filter (130)
+- BatchEvalPython (129)
+- Project (128)
+- BroadcastHashJoin Inner BuildLeft (127)
:- BroadcastExchange (120)
: +- Project (119)
: +- SortMergeJoin LeftAnti (118)
: :- Sort (103)
: : +- Exchange (102)
: : +- Project (101)
: : +- BatchEvalPython (100)
: : +- Project (99)
: : +- Filter (98)
: : +- BatchEvalPython (97)
: : +- Filter (96)
: : +- InMemoryTableScan (93)
: : +- InMemoryRelation (94)
: : +- AdaptiveSparkPlan (95)
: : +- Project (9)
: : +- Filter (8)
: : +- InMemoryTableScan (5)
: : +- InMemoryRelation (6)
: : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: +- Sort (117)
: +- Exchange (116)
: +- Project (115)
: +- BroadcastHashJoin Inner BuildLeft (114)
: :- BroadcastExchange (108)
: : +- Filter (107)
: : +- InMemoryTableScan (104)
: : +- InMemoryRelation (105)
: : +- AdaptiveSparkPlan (106)
: : +- Project (9)
: : +- Filter (8)
: : +- InMemoryTableScan (5)
: : +- InMemoryRelation (6)
: : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: +- Project (113)
: +- Filter (112)
: +- InMemoryTableScan (109)
: +- InMemoryRelation (110)
: +- AdaptiveSparkPlan (111)
: +- Filter (18)
: +- InMemoryTableScan (15)
: +- InMemoryRelation (16)
: +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (17)
+- Project (126)
+- BatchEvalPython (125)
+- Filter (124)
+- InMemoryTableScan (121)
+- InMemoryRelation (122)
+- AdaptiveSparkPlan (123)
+- Filter (18)
+- InMemoryTableScan (15)
+- InMemoryRelation (16)
+- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (17)
+- == Initial Plan ==
HashAggregate (154)
+- Exchange (153)
+- HashAggregate (152)
+- InMemoryTableScan (1)
+- InMemoryRelation (2)
+- AdaptiveSparkPlan (145)
+- Union (144)
:- Project (23)
: +- BroadcastHashJoin Inner BuildLeft (22)
: :- BroadcastExchange (12)
: : +- Filter (11)
: : +- InMemoryTableScan (3)
: : +- InMemoryRelation (4)
: : +- AdaptiveSparkPlan (10)
: : +- Project (9)
: : +- Filter (8)
: : +- InMemoryTableScan (5)
: : +- InMemoryRelation (6)
: : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: +- Project (21)
: +- Filter (20)
: +- InMemoryTableScan (13)
: +- InMemoryRelation (14)
: +- AdaptiveSparkPlan (19)
: +- Filter (18)
: +- InMemoryTableScan (15)
: +- InMemoryRelation (16)
: +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (17)
:- Project (72)
: +- Filter (71)
: +- Window (70)
: +- WindowGroupLimit (69)
: +- Sort (68)
: +- Exchange (67)
: +- WindowGroupLimit (66)
: +- Sort (65)
: +- Project (64)
: +- BatchEvalPython (63)
: +- Project (62)
: +- Filter (61)
: +- BatchEvalPython (60)
: +- Project (59)
: +- BroadcastHashJoin Inner BuildLeft (58)
: :- BroadcastExchange (51)
: : +- Project (50)
: : +- SortMergeJoin LeftAnti (49)
: : :- Sort (34)
: : : +- Exchange (33)
: : : +- Project (32)
: : : +- BatchEvalPython (31)
: : : +- Project (30)
: : : +- Filter (29)
: : : +- BatchEvalPython (28)
: : : +- Filter (27)
: : : +- InMemoryTableScan (24)
: : : +- InMemoryRelation (25)
: : : +- AdaptiveSparkPlan (26)
: : : +- Project (9)
: : : +- Filter (8)
: : : +- InMemoryTableScan (5)
: : : +- InMemoryRelation (6)
: : : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: : +- Sort (48)
: : +- Exchange (47)
: : +- Project (46)
: : +- BroadcastHashJoin Inner BuildLeft (45)
: : :- BroadcastExchange (39)
: : : +- Filter (38)
: : : +- InMemoryTableScan (35)
: : : +- InMemoryRelation (36)
: : : +- AdaptiveSparkPlan (37)
: : : +- Project (9)
: : : +- Filter (8)
: : : +- InMemoryTableScan (5)
: : : +- InMemoryRelation (6)
: : : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: : +- Project (44)
: : +- Filter (43)
: : +- InMemoryTableScan (40)
: : +- InMemoryRelation (41)
: : +- AdaptiveSparkPlan (42)
: : +- Filter (18)
: : +- InMemoryTableScan (15)
: : +- InMemoryRelation (16)
: : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (17)
: +- Project (57)
: +- BatchEvalPython (56)
: +- Filter (55)
: +- InMemoryTableScan (52)
: +- InMemoryRelation (53)
: +- AdaptiveSparkPlan (54)
: +- Filter (18)
: +- InMemoryTableScan (15)
: +- InMemoryRelation (16)
: +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (17)
+- Project (143)
+- SortMergeJoin LeftAnti (142)
:- SortMergeJoin LeftAnti (92)
: :- Sort (77)
: : +- Exchange (76)
: : +- InMemoryTableScan (73)
: : +- InMemoryRelation (74)
: : +- AdaptiveSparkPlan (75)
: : +- Project (9)
: : +- Filter (8)
: : +- InMemoryTableScan (5)
: : +- InMemoryRelation (6)
: : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: +- Sort (91)
: +- Exchange (90)
: +- Project (89)
: +- BroadcastHashJoin Inner BuildLeft (88)
: :- BroadcastExchange (82)
: : +- Filter (81)
: : +- InMemoryTableScan (78)
: : +- InMemoryRelation (79)
: : +- AdaptiveSparkPlan (80)
: : +- Project (9)
: : +- Filter (8)
: : +- InMemoryTableScan (5)
: : +- InMemoryRelation (6)
: : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: +- Project (87)
: +- Filter (86)
: +- InMemoryTableScan (83)
: +- InMemoryRelation (84)
: +- AdaptiveSparkPlan (85)
: +- Filter (18)
: +- InMemoryTableScan (15)
: +- InMemoryRelation (16)
: +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (17)
+- Project (141)
+- Filter (140)
+- Window (139)
+- WindowGroupLimit (138)
+- Sort (137)
+- Exchange (136)
+- WindowGroupLimit (135)
+- Sort (134)
+- Project (133)
+- BatchEvalPython (132)
+- Project (131)
+- Filter (130)
+- BatchEvalPython (129)
+- Project (128)
+- BroadcastHashJoin Inner BuildLeft (127)
:- BroadcastExchange (120)
: +- Project (119)
: +- SortMergeJoin LeftAnti (118)
: :- Sort (103)
: : +- Exchange (102)
: : +- Project (101)
: : +- BatchEvalPython (100)
: : +- Project (99)
: : +- Filter (98)
: : +- BatchEvalPython (97)
: : +- Filter (96)
: : +- InMemoryTableScan (93)
: : +- InMemoryRelation (94)
: : +- AdaptiveSparkPlan (95)
: : +- Project (9)
: : +- Filter (8)
: : +- InMemoryTableScan (5)
: : +- InMemoryRelation (6)
: : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: +- Sort (117)
: +- Exchange (116)
: +- Project (115)
: +- BroadcastHashJoin Inner BuildLeft (114)
: :- BroadcastExchange (108)
: : +- Filter (107)
: : +- InMemoryTableScan (104)
: : +- InMemoryRelation (105)
: : +- AdaptiveSparkPlan (106)
: : +- Project (9)
: : +- Filter (8)
: : +- InMemoryTableScan (5)
: : +- InMemoryRelation (6)
: : +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (7)
: +- Project (113)
: +- Filter (112)
: +- InMemoryTableScan (109)
: +- InMemoryRelation (110)
: +- AdaptiveSparkPlan (111)
: +- Filter (18)
: +- InMemoryTableScan (15)
: +- InMemoryRelation (16)
: +- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (17)
+- Project (126)
+- BatchEvalPython (125)
+- Filter (124)
+- InMemoryTableScan (121)
+- InMemoryRelation (122)
+- AdaptiveSparkPlan (123)
+- Filter (18)
+- InMemoryTableScan (15)
+- InMemoryRelation (16)
+- * Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] (17)
(1) InMemoryTableScan
Output [1]: [match_reason#10397914]
Arguments: [match_reason#10397914]
(2) InMemoryRelation
Arguments: [brand_context_id#8983060, supplier_id#8983061, declaring_supplier_id#8983067, supplier_type#8983066, canonical_id#10397736, is_primary#10397910, configured_threshold#10397911, match_percentage#10397912, confidence_score#10397913, match_reason#10397914], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@7795a5ad,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
Union
:- *(26) Project [brand_context_id#8983060, supplier_id#8983061, declaring_supplier_id#8983067, supplier_type#8983066, canonical_id#10397736, 0 AS is_primary#10397910, threshold#8983070 AS configured_threshold#10397911, 100.0 AS match_percentage#10397912, 1.0 AS confidence_score#10397913, exact_id_match AS match_reason#10397914]
: +- *(26) BroadcastHashJoin [brand_context_id#8983060, supplier_id#8983061], [brand_context_id#10397728, supplier_id#10397729], Inner, BuildLeft, false
: :- BroadcastQueryStage 13
: : +- BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, false], input[1, string, false]),false), [plan_id=357354]
: : +- *(1) Filter (isnotnull(brand_context_id#8983060) AND isnotnull(supplier_id#8983061))
: : +- TableCacheQueryStage 0
: : +- InMemoryTableScan [brand_context_id#8983060, supplier_id#8983061, supplier_type#8983066, declaring_supplier_id#8983067, threshold#8983070], [isnotnull(brand_context_id#8983060), isnotnull(supplier_id#8983061)]
: : +- InMemoryRelation [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, threshold#8983070, create_date#8983071], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- AdaptiveSparkPlan isFinalPlan=false
: : +- Project [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, threshold#8983070, create_date#8983071]
: : +- Filter isnull(canonical_id#8983068)
: : +- InMemoryTableScan [brand_context_id#8983060, canonical_id#8983068, city#8983063, country#8983065, create_date#8983071, declaring_supplier_id#8983067, entity_name#8983062, state#8983064, supplier_id#8983061, supplier_type#8983066, threshold#8983070], [isnull(canonical_id#8983068)]
: : +- InMemoryRelation [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, canonical_id#8983068, is_primary#8983069, threshold#8983070, create_date#8983071], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#8983060,supplier_id#8983061,entity_name#8983062,city#8983063,state#8983064,country#8983065,supplier_type#8983066,declaring_supplier_id#8983067,canonical_id#8983068,is_primary#8983069,threshold#8983070,create_date#8983071] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
: +- *(26) Project [brand_context_id#10397728, supplier_id#10397729, canonical_id#10397736]
: +- *(26) Filter ((((isnotnull(is_primary#10397737) AND is_primary#10397737) AND isnotnull(canonical_id#10397736)) AND isnotnull(brand_context_id#10397728)) AND isnotnull(supplier_id#10397729))
: +- TableCacheQueryStage 1
: +- InMemoryTableScan [brand_context_id#10397728, canonical_id#10397736, is_primary#10397737, supplier_id#10397729], [isnotnull(is_primary#10397737), is_primary#10397737, isnotnull(canonical_id#10397736), isnotnull(brand_context_id#10397728), isnotnull(supplier_id#10397729)]
: +- InMemoryRelation [brand_context_id#10397728, supplier_id#10397729, canonical_id#10397736, is_primary#10397737, entity_name#10397730, city#10397731, state#10397732, country#10397733, threshold#10397738, create_date#10397739], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- AdaptiveSparkPlan isFinalPlan=false
: +- Filter isnotnull(canonical_id#8983068)
: +- InMemoryTableScan [brand_context_id#8983060, supplier_id#8983061, canonical_id#8983068, is_primary#8983069, entity_name#8983062, city#8983063, state#8983064, country#8983065, threshold#8983070, create_date#8983071], [isnotnull(canonical_id#8983068)]
: +- InMemoryRelation [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, canonical_id#8983068, is_primary#8983069, threshold#8983070, create_date#8983071], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#8983060,supplier_id#8983061,entity_name#8983062,city#8983063,state#8983064,country#8983065,supplier_type#8983066,declaring_supplier_id#8983067,canonical_id#8983068,is_primary#8983069,threshold#8983070,create_date#8983071] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
:- *(28) Project [u_brand_context_id#10399066 AS brand_context_id#10399187, u_supplier_id#10399067 AS supplier_id#10399188, u_declaring_supplier_id#10399068 AS declaring_supplier_id#10399189, u_supplier_type#10399069 AS supplier_type#10399190, e_canonical_id#10399090 AS canonical_id#10399191, 0 AS is_primary#10399192, u_threshold#10399070 AS configured_threshold#10399193, cast((similarity#10399147 * 100.0) as double) AS match_percentage#10399300, cast(similarity#10399147 as double) AS confidence_score#10399301, matched_existing AS match_reason#10399196]
: +- *(28) Filter (rank#10399167 = 1)
: +- Window [row_number() windowspecdefinition(u_brand_context_id#10399066, u_supplier_id#10399067, similarity#10399147 DESC NULLS LAST, e_create_date#10399093 ASC NULLS FIRST, e_supplier_id#10399089 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank#10399167], [u_brand_context_id#10399066, u_supplier_id#10399067], [similarity#10399147 DESC NULLS LAST, e_create_date#10399093 ASC NULLS FIRST, e_supplier_id#10399089 ASC NULLS FIRST]
: +- WindowGroupLimit [u_brand_context_id#10399066, u_supplier_id#10399067], [similarity#10399147 DESC NULLS LAST, e_create_date#10399093 ASC NULLS FIRST, e_supplier_id#10399089 ASC NULLS FIRST], row_number(), 1, Final
: +- *(27) Sort [u_brand_context_id#10399066 ASC NULLS FIRST, u_supplier_id#10399067 ASC NULLS FIRST, similarity#10399147 DESC NULLS LAST, e_create_date#10399093 ASC NULLS FIRST, e_supplier_id#10399089 ASC NULLS FIRST], false, 0
: +- ShuffleQueryStage 25
: +- Exchange hashpartitioning(u_brand_context_id#10399066, u_supplier_id#10399067, 10), ENSURE_REQUIREMENTS, [plan_id=360615]
: +- WindowGroupLimit [u_brand_context_id#10399066, u_supplier_id#10399067], [similarity#10399147 DESC NULLS LAST, e_create_date#10399093 ASC NULLS FIRST, e_supplier_id#10399089 ASC NULLS FIRST], row_number(), 1, Partial
: +- *(19) Sort [u_brand_context_id#10399066 ASC NULLS FIRST, u_supplier_id#10399067 ASC NULLS FIRST, similarity#10399147 DESC NULLS LAST, e_create_date#10399093 ASC NULLS FIRST, e_supplier_id#10399089 ASC NULLS FIRST], false, 0
: +- *(19) Project [u_brand_context_id#10399066, u_supplier_id#10399067, u_declaring_supplier_id#10399068, u_supplier_type#10399069, u_threshold#10399070, e_supplier_id#10399089, e_canonical_id#10399090, e_create_date#10399093, pythonUDF0#10400695 AS similarity#10399147]
: +- BatchEvalPython [calculate_similarity(u_normalized_name#10399071, e_normalized_name#10399091)#10399146], [pythonUDF0#10400695]
: +- *(18) Project [u_brand_context_id#10399066, u_supplier_id#10399067, u_declaring_supplier_id#10399068, u_supplier_type#10399069, u_threshold#10399070, u_normalized_name#10399071, e_supplier_id#10399089, e_canonical_id#10399090, e_normalized_name#10399091, e_create_date#10399093]
: +- *(18) Filter (cast(pythonUDF0#10400694 as double) >= u_threshold#10399070)
: +- BatchEvalPython [calculate_similarity(u_normalized_name#10399071, e_normalized_name#10399091)#10399146], [pythonUDF0#10400694]
: +- *(17) Project [u_brand_context_id#10399066, u_supplier_id#10399067, u_declaring_supplier_id#10399068, u_supplier_type#10399069, u_threshold#10399070, u_normalized_name#10399071, e_supplier_id#10399089, e_canonical_id#10399090, e_normalized_name#10399091, e_create_date#10399093]
: +- *(17) BroadcastHashJoin [u_brand_context_id#10399066, u_location_key#10399075], [e_brand_context_id#10399088, e_location_key#10399092], Inner, BuildLeft, false
: :- BroadcastQueryStage 23
: : +- BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, true], input[6, string, false]),false), [plan_id=359934]
: : +- *(14) Project [brand_context_id#10399276 AS u_brand_context_id#10399066, supplier_id#10399277 AS u_supplier_id#10399067, declaring_supplier_id#10399283 AS u_declaring_supplier_id#10399068, supplier_type#10399282 AS u_supplier_type#10399069, threshold#10399286 AS u_threshold#10399070, normalized_name#10395248 AS u_normalized_name#10399071, location_key#10395305 AS u_location_key#10399075]
: : +- AQEShuffleRead coalesced
: : +- ShuffleQueryStage 14
: : +- Exchange hashpartitioning(brand_context_id#10399276, supplier_id#10399277, 10), ENSURE_REQUIREMENTS, [plan_id=357436]
: : +- *(4) Project [brand_context_id#10399276, supplier_id#10399277, supplier_type#10399282, declaring_supplier_id#10399283, threshold#10399286, pythonUDF0#10400686 AS normalized_name#10395248, concat_ws(|, coalesce(pythonUDF1#10400687, ), coalesce(pythonUDF2#10400688, ), coalesce(pythonUDF3#10400689, )) AS location_key#10395305]
: : +- BatchEvalPython [normalize_name(entity_name#10399278)#10395247, normalize_location(city#10399279)#10395260, normalize_location(state#10399280)#10395274, normalize_location(country#10399281)#10395289], [pythonUDF0#10400686, pythonUDF1#10400687, pythonUDF2#10400688, pythonUDF3#10400689]
: : +- *(3) Project [brand_context_id#10399276, supplier_id#10399277, entity_name#10399278, city#10399279, state#10399280, country#10399281, supplier_type#10399282, declaring_supplier_id#10399283, threshold#10399286]
: : +- *(3) Filter ((isnotnull(pythonUDF0#10400683) AND isnotnull(pythonUDF1#10400684)) AND isnotnull(pythonUDF2#10400685))
: : +- BatchEvalPython [normalize_location(city#10399279)#10395260, normalize_location(state#10399280)#10395274, normalize_location(country#10399281)#10395289], [pythonUDF0#10400683, pythonUDF1#10400684, pythonUDF2#10400685]
: : +- *(2) Filter (isnotnull(threshold#10399286) AND isnotnull(brand_context_id#10399276))
: : +- TableCacheQueryStage 2
: : +- InMemoryTableScan [brand_context_id#10399276, supplier_id#10399277, entity_name#10399278, city#10399279, state#10399280, country#10399281, supplier_type#10399282, declaring_supplier_id#10399283, threshold#10399286], [isnotnull(threshold#10399286), isnotnull(brand_context_id#10399276)]
: : +- InMemoryRelation [brand_context_id#10399276, supplier_id#10399277, entity_name#10399278, city#10399279, state#10399280, country#10399281, supplier_type#10399282, declaring_supplier_id#10399283, threshold#10399286, create_date#10399287], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- AdaptiveSparkPlan isFinalPlan=false
: : +- Project [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, threshold#8983070, create_date#8983071]
: : +- Filter isnull(canonical_id#8983068)
: : +- InMemoryTableScan [brand_context_id#8983060, canonical_id#8983068, city#8983063, country#8983065, create_date#8983071, declaring_supplier_id#8983067, entity_name#8983062, state#8983064, supplier_id#8983061, supplier_type#8983066, threshold#8983070], [isnull(canonical_id#8983068)]
: : +- InMemoryRelation [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, canonical_id#8983068, is_primary#8983069, threshold#8983070, create_date#8983071], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#8983060,supplier_id#8983061,entity_name#8983062,city#8983063,state#8983064,country#8983065,supplier_type#8983066,declaring_supplier_id#8983067,canonical_id#8983068,is_primary#8983069,threshold#8983070,create_date#8983071] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
: +- *(17) Project [brand_context_id#10399100 AS e_brand_context_id#10399088, supplier_id#10399101 AS e_supplier_id#10399089, canonical_id#10399108 AS e_canonical_id#10399090, pythonUDF0#10400690 AS e_normalized_name#10399091, concat_ws(|, coalesce(pythonUDF1#10400691, ), coalesce(pythonUDF2#10400692, ), coalesce(pythonUDF3#10400693, )) AS e_location_key#10399092, create_date#10399111 AS e_create_date#10399093]
: +- BatchEvalPython [normalize_name(entity_name#10399102)#10395321, normalize_location(city#10399103)#10395334, normalize_location(state#10399104)#10395348, normalize_location(country#10399105)#10395363], [pythonUDF0#10400690, pythonUDF1#10400691, pythonUDF2#10400692, pythonUDF3#10400693]
: +- *(16) Filter isnotnull(brand_context_id#10399100)
: +- TableCacheQueryStage 5
: +- InMemoryTableScan [brand_context_id#10399100, supplier_id#10399101, canonical_id#10399108, entity_name#10399102, city#10399103, state#10399104, country#10399105, create_date#10399111], [isnotnull(brand_context_id#10399100)]
: +- InMemoryRelation [brand_context_id#10399100, supplier_id#10399101, canonical_id#10399108, is_primary#10399109, entity_name#10399102, city#10399103, state#10399104, country#10399105, threshold#10399110, create_date#10399111], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- AdaptiveSparkPlan isFinalPlan=false
: +- Filter isnotnull(canonical_id#8983068)
: +- InMemoryTableScan [brand_context_id#8983060, supplier_id#8983061, canonical_id#8983068, is_primary#8983069, entity_name#8983062, city#8983063, state#8983064, country#8983065, threshold#8983070, create_date#8983071], [isnotnull(canonical_id#8983068)]
: +- InMemoryRelation [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, canonical_id#8983068, is_primary#8983069, threshold#8983070, create_date#8983071], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#8983060,supplier_id#8983061,entity_name#8983062,city#8983063,state#8983064,country#8983065,supplier_type#8983066,declaring_supplier_id#8983067,canonical_id#8983068,is_primary#8983069,threshold#8983070,create_date#8983071] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- *(29) Project [brand_context_id#10399312, supplier_id#10399313, declaring_supplier_id#10399319, supplier_type#10399318, supplier_id#10399313 AS canonical_id#10399260, 1 AS is_primary#10399261, threshold#10399322 AS configured_threshold#10399262, 0.0 AS match_percentage#10399263, 0.0 AS confidence_score#10399264, new_canonical AS match_reason#10399265]
+- *(29) BroadcastHashJoin [brand_context_id#10399312, supplier_id#10399313], [brand_context_id#10399187, supplier_id#10399188], LeftAnti, BuildRight, false
:- ShuffleQueryStage 16
: +- Exchange hashpartitioning(brand_context_id#10399312, supplier_id#10399313, 10), ENSURE_REQUIREMENTS, [plan_id=356940]
: +- TableCacheQueryStage 6
: +- InMemoryTableScan [brand_context_id#10399312, supplier_id#10399313, supplier_type#10399318, declaring_supplier_id#10399319, threshold#10399322]
: +- InMemoryRelation [brand_context_id#10399312, supplier_id#10399313, entity_name#10399314, city#10399315, state#10399316, country#10399317, supplier_type#10399318, declaring_supplier_id#10399319, threshold#10399322, create_date#10399323], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- AdaptiveSparkPlan isFinalPlan=false
: +- Project [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, threshold#8983070, create_date#8983071]
: +- Filter isnull(canonical_id#8983068)
: +- InMemoryTableScan [brand_context_id#8983060, canonical_id#8983068, city#8983063, country#8983065, create_date#8983071, declaring_supplier_id#8983067, entity_name#8983062, state#8983064, supplier_id#8983061, supplier_type#8983066, threshold#8983070], [isnull(canonical_id#8983068)]
: +- InMemoryRelation [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, canonical_id#8983068, is_primary#8983069, threshold#8983070, create_date#8983071], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#8983060,supplier_id#8983061,entity_name#8983062,city#8983063,state#8983064,country#8983065,supplier_type#8983066,declaring_supplier_id#8983067,canonical_id#8983068,is_primary#8983069,threshold#8983070,create_date#8983071] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- BroadcastQueryStage 27
+- BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true]),false), [plan_id=361051]
+- *(25) Project [u_brand_context_id#10399066 AS brand_context_id#10399187, u_supplier_id#10399067 AS supplier_id#10399188]
+- *(25) Filter (rank#10399167 = 1)
+- Window [row_number() windowspecdefinition(u_brand_context_id#10399066, u_supplier_id#10399067, similarity#10399147 DESC NULLS LAST, e_create_date#10399093 ASC NULLS FIRST, e_supplier_id#10399089 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank#10399167], [u_brand_context_id#10399066, u_supplier_id#10399067], [similarity#10399147 DESC NULLS LAST, e_create_date#10399093 ASC NULLS FIRST, e_supplier_id#10399089 ASC NULLS FIRST]
+- WindowGroupLimit [u_brand_context_id#10399066, u_supplier_id#10399067], [similarity#10399147 DESC NULLS LAST, e_create_date#10399093 ASC NULLS FIRST, e_supplier_id#10399089 ASC NULLS FIRST], row_number(), 1, Final
+- *(24) Sort [u_brand_context_id#10399066 ASC NULLS FIRST, u_supplier_id#10399067 ASC NULLS FIRST, similarity#10399147 DESC NULLS LAST, e_create_date#10399093 ASC NULLS FIRST, e_supplier_id#10399089 ASC NULLS FIRST], false, 0
+- AQEShuffleRead coalesced
+- ShuffleQueryStage 26
+- Exchange hashpartitioning(u_brand_context_id#10399066, u_supplier_id#10399067, 10), ENSURE_REQUIREMENTS, [plan_id=360789]
+- WindowGroupLimit [u_brand_context_id#10399066, u_supplier_id#10399067], [similarity#10399147 DESC NULLS LAST, e_create_date#10399093 ASC NULLS FIRST, e_supplier_id#10399089 ASC NULLS FIRST], row_number(), 1, Partial
+- *(23) Sort [u_brand_context_id#10399066 ASC NULLS FIRST, u_supplier_id#10399067 ASC NULLS FIRST, similarity#10399147 DESC NULLS LAST, e_create_date#10399093 ASC NULLS FIRST, e_supplier_id#10399089 ASC NULLS FIRST], false, 0
+- *(23) Project [u_brand_context_id#10399066, u_supplier_id#10399067, e_supplier_id#10399089, e_create_date#10399093, pythonUDF0#10400708 AS similarity#10399147]
+- BatchEvalPython [calculate_similarity(u_normalized_name#10399071, e_normalized_name#10399091)#10399146], [pythonUDF0#10400708]
+- *(22) Project [u_brand_context_id#10399066, u_supplier_id#10399067, u_normalized_name#10399071, e_supplier_id#10399089, e_normalized_name#10399091, e_create_date#10399093]
+- *(22) Filter (cast(pythonUDF0#10400707 as double) >= u_threshold#10399070)
+- BatchEvalPython [calculate_similarity(u_normalized_name#10399071, e_normalized_name#10399091)#10399146], [pythonUDF0#10400707]
+- *(21) Project [u_brand_context_id#10399066, u_supplier_id#10399067, u_threshold#10399070, u_normalized_name#10399071, e_supplier_id#10399089, e_normalized_name#10399091, e_create_date#10399093]
+- *(21) BroadcastHashJoin [u_brand_context_id#10399066, u_location_key#10399075], [e_brand_context_id#10399088, e_location_key#10399092], Inner, BuildLeft, false
:- BroadcastQueryStage 24
: +- BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, true], input[4, string, false]),false), [plan_id=360194]
: +- *(15) Project [brand_context_id#10399209 AS u_brand_context_id#10399066, supplier_id#10399210 AS u_supplier_id#10399067, threshold#10399219 AS u_threshold#10399070, normalized_name#10395248 AS u_normalized_name#10399071, location_key#10395305 AS u_location_key#10399075]
: +- AQEShuffleRead coalesced
: +- ShuffleQueryStage 18
: +- Exchange hashpartitioning(brand_context_id#10399209, supplier_id#10399210, 10), ENSURE_REQUIREMENTS, [plan_id=357594]
: +- *(9) Project [brand_context_id#10399209, supplier_id#10399210, threshold#10399219, pythonUDF0#10400699 AS normalized_name#10395248, concat_ws(|, coalesce(pythonUDF1#10400700, ), coalesce(pythonUDF2#10400701, ), coalesce(pythonUDF3#10400702, )) AS location_key#10395305]
: +- BatchEvalPython [normalize_name(entity_name#10399211)#10395247, normalize_location(city#10399212)#10395260, normalize_location(state#10399213)#10395274, normalize_location(country#10399214)#10395289], [pythonUDF0#10400699, pythonUDF1#10400700, pythonUDF2#10400701, pythonUDF3#10400702]
: +- *(8) Project [brand_context_id#10399209, supplier_id#10399210, entity_name#10399211, city#10399212, state#10399213, country#10399214, threshold#10399219]
: +- *(8) Filter ((isnotnull(pythonUDF0#10400696) AND isnotnull(pythonUDF1#10400697)) AND isnotnull(pythonUDF2#10400698))
: +- BatchEvalPython [normalize_location(city#10399212)#10395260, normalize_location(state#10399213)#10395274, normalize_location(country#10399214)#10395289], [pythonUDF0#10400696, pythonUDF1#10400697, pythonUDF2#10400698]
: +- *(7) Filter ((isnotnull(threshold#10399219) AND isnotnull(brand_context_id#10399209)) AND isnotnull(supplier_id#10399210))
: +- TableCacheQueryStage 9
: +- InMemoryTableScan [brand_context_id#10399209, supplier_id#10399210, entity_name#10399211, city#10399212, state#10399213, country#10399214, threshold#10399219], [isnotnull(threshold#10399219), isnotnull(brand_context_id#10399209), isnotnull(supplier_id#10399210)]
: +- InMemoryRelation [brand_context_id#10399209, supplier_id#10399210, entity_name#10399211, city#10399212, state#10399213, country#10399214, supplier_type#10399215, declaring_supplier_id#10399216, threshold#10399219, create_date#10399220], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- AdaptiveSparkPlan isFinalPlan=false
: +- Project [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, threshold#8983070, create_date#8983071]
: +- Filter isnull(canonical_id#8983068)
: +- InMemoryTableScan [brand_context_id#8983060, canonical_id#8983068, city#8983063, country#8983065, create_date#8983071, declaring_supplier_id#8983067, entity_name#8983062, state#8983064, supplier_id#8983061, supplier_type#8983066, threshold#8983070], [isnull(canonical_id#8983068)]
: +- InMemoryRelation [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, canonical_id#8983068, is_primary#8983069, threshold#8983070, create_date#8983071], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#8983060,supplier_id#8983061,entity_name#8983062,city#8983063,state#8983064,country#8983065,supplier_type#8983066,declaring_supplier_id#8983067,canonical_id#8983068,is_primary#8983069,threshold#8983070,create_date#8983071] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- *(21) Project [brand_context_id#10399348 AS e_brand_context_id#10399088, supplier_id#10399349 AS e_supplier_id#10399089, pythonUDF0#10400703 AS e_normalized_name#10399091, concat_ws(|, coalesce(pythonUDF1#10400704, ), coalesce(pythonUDF2#10400705, ), coalesce(pythonUDF3#10400706, )) AS e_location_key#10399092, create_date#10399359 AS e_create_date#10399093]
+- BatchEvalPython [normalize_name(entity_name#10399350)#10395321, normalize_location(city#10399351)#10395334, normalize_location(state#10399352)#10395348, normalize_location(country#10399353)#10395363], [pythonUDF0#10400703, pythonUDF1#10400704, pythonUDF2#10400705, pythonUDF3#10400706]
+- *(20) Filter isnotnull(brand_context_id#10399348)
+- TableCacheQueryStage 12
+- InMemoryTableScan [brand_context_id#10399348, supplier_id#10399349, entity_name#10399350, city#10399351, state#10399352, country#10399353, create_date#10399359], [isnotnull(brand_context_id#10399348)]
+- InMemoryRelation [brand_context_id#10399348, supplier_id#10399349, canonical_id#10399356, is_primary#10399357, entity_name#10399350, city#10399351, state#10399352, country#10399353, threshold#10399358, create_date#10399359], StorageLevel(disk, memory, deserialized, 1 replicas)
+- AdaptiveSparkPlan isFinalPlan=false
+- Filter isnotnull(canonical_id#8983068)
+- InMemoryTableScan [brand_context_id#8983060, supplier_id#8983061, canonical_id#8983068, is_primary#8983069, entity_name#8983062, city#8983063, state#8983064, country#8983065, threshold#8983070, create_date#8983071], [isnotnull(canonical_id#8983068)]
+- InMemoryRelation [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, canonical_id#8983068, is_primary#8983069, threshold#8983070, create_date#8983071], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#8983060,supplier_id#8983061,entity_name#8983062,city#8983063,state#8983064,country#8983065,supplier_type#8983066,declaring_supplier_id#8983067,canonical_id#8983068,is_primary#8983069,threshold#8983070,create_date#8983071] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Union
:- Project [brand_context_id#8983060, supplier_id#8983061, declaring_supplier_id#8983067, supplier_type#8983066, canonical_id#10397736, 0 AS is_primary#10397910, threshold#8983070 AS configured_threshold#10397911, 100.0 AS match_percentage#10397912, 1.0 AS confidence_score#10397913, exact_id_match AS match_reason#10397914]
: +- BroadcastHashJoin [brand_context_id#8983060, supplier_id#8983061], [brand_context_id#10397728, supplier_id#10397729], Inner, BuildLeft, false
: :- BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, false], input[1, string, false]),false), [plan_id=356628]
: : +- Filter (isnotnull(brand_context_id#8983060) AND isnotnull(supplier_id#8983061))
: : +- InMemoryTableScan [brand_context_id#8983060, supplier_id#8983061, supplier_type#8983066, declaring_supplier_id#8983067, threshold#8983070], [isnotnull(brand_context_id#8983060), isnotnull(supplier_id#8983061)]
: : +- InMemoryRelation [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, threshold#8983070, create_date#8983071], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- AdaptiveSparkPlan isFinalPlan=false
: : +- Project [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, threshold#8983070, create_date#8983071]
: : +- Filter isnull(canonical_id#8983068)
: : +- InMemoryTableScan [brand_context_id#8983060, canonical_id#8983068, city#8983063, country#8983065, create_date#8983071, declaring_supplier_id#8983067, entity_name#8983062, state#8983064, supplier_id#8983061, supplier_type#8983066, threshold#8983070], [isnull(canonical_id#8983068)]
: : +- InMemoryRelation [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, canonical_id#8983068, is_primary#8983069, threshold#8983070, create_date#8983071], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#8983060,supplier_id#8983061,entity_name#8983062,city#8983063,state#8983064,country#8983065,supplier_type#8983066,declaring_supplier_id#8983067,canonical_id#8983068,is_primary#8983069,threshold#8983070,create_date#8983071] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
: +- Project [brand_context_id#10397728, supplier_id#10397729, canonical_id#10397736]
: +- Filter ((((isnotnull(is_primary#10397737) AND is_primary#10397737) AND isnotnull(canonical_id#10397736)) AND isnotnull(brand_context_id#10397728)) AND isnotnull(supplier_id#10397729))
: +- InMemoryTableScan [brand_context_id#10397728, canonical_id#10397736, is_primary#10397737, supplier_id#10397729], [isnotnull(is_primary#10397737), is_primary#10397737, isnotnull(canonical_id#10397736), isnotnull(brand_context_id#10397728), isnotnull(supplier_id#10397729)]
: +- InMemoryRelation [brand_context_id#10397728, supplier_id#10397729, canonical_id#10397736, is_primary#10397737, entity_name#10397730, city#10397731, state#10397732, country#10397733, threshold#10397738, create_date#10397739], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- AdaptiveSparkPlan isFinalPlan=false
: +- Filter isnotnull(canonical_id#8983068)
: +- InMemoryTableScan [brand_context_id#8983060, supplier_id#8983061, canonical_id#8983068, is_primary#8983069, entity_name#8983062, city#8983063, state#8983064, country#8983065, threshold#8983070, create_date#8983071], [isnotnull(canonical_id#8983068)]
: +- InMemoryRelation [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, canonical_id#8983068, is_primary#8983069, threshold#8983070, create_date#8983071], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#8983060,supplier_id#8983061,entity_name#8983062,city#8983063,state#8983064,country#8983065,supplier_type#8983066,declaring_supplier_id#8983067,canonical_id#8983068,is_primary#8983069,threshold#8983070,create_date#8983071] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
:- Project [u_brand_context_id#10399066 AS brand_context_id#10399187, u_supplier_id#10399067 AS supplier_id#10399188, u_declaring_supplier_id#10399068 AS declaring_supplier_id#10399189, u_supplier_type#10399069 AS supplier_type#10399190, e_canonical_id#10399090 AS canonical_id#10399191, 0 AS is_primary#10399192, u_threshold#10399070 AS configured_threshold#10399193, cast((similarity#10399147 * 100.0) as double) AS match_percentage#10399300, cast(similarity#10399147 as double) AS confidence_score#10399301, matched_existing AS match_reason#10399196]
: +- Filter (rank#10399167 = 1)
: +- Window [row_number() windowspecdefinition(u_brand_context_id#10399066, u_supplier_id#10399067, similarity#10399147 DESC NULLS LAST, e_create_date#10399093 ASC NULLS FIRST, e_supplier_id#10399089 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank#10399167], [u_brand_context_id#10399066, u_supplier_id#10399067], [similarity#10399147 DESC NULLS LAST, e_create_date#10399093 ASC NULLS FIRST, e_supplier_id#10399089 ASC NULLS FIRST]
: +- WindowGroupLimit [u_brand_context_id#10399066, u_supplier_id#10399067], [similarity#10399147 DESC NULLS LAST, e_create_date#10399093 ASC NULLS FIRST, e_supplier_id#10399089 ASC NULLS FIRST], row_number(), 1, Final
: +- Sort [u_brand_context_id#10399066 ASC NULLS FIRST, u_supplier_id#10399067 ASC NULLS FIRST, similarity#10399147 DESC NULLS LAST, e_create_date#10399093 ASC NULLS FIRST, e_supplier_id#10399089 ASC NULLS FIRST], false, 0
: +- Exchange hashpartitioning(u_brand_context_id#10399066, u_supplier_id#10399067, 10), ENSURE_REQUIREMENTS, [plan_id=356655]
: +- WindowGroupLimit [u_brand_context_id#10399066, u_supplier_id#10399067], [similarity#10399147 DESC NULLS LAST, e_create_date#10399093 ASC NULLS FIRST, e_supplier_id#10399089 ASC NULLS FIRST], row_number(), 1, Partial
: +- Sort [u_brand_context_id#10399066 ASC NULLS FIRST, u_supplier_id#10399067 ASC NULLS FIRST, similarity#10399147 DESC NULLS LAST, e_create_date#10399093 ASC NULLS FIRST, e_supplier_id#10399089 ASC NULLS FIRST], false, 0
: +- Project [u_brand_context_id#10399066, u_supplier_id#10399067, u_declaring_supplier_id#10399068, u_supplier_type#10399069, u_threshold#10399070, e_supplier_id#10399089, e_canonical_id#10399090, e_create_date#10399093, pythonUDF0#10400695 AS similarity#10399147]
: +- BatchEvalPython [calculate_similarity(u_normalized_name#10399071, e_normalized_name#10399091)#10399146], [pythonUDF0#10400695]
: +- Project [u_brand_context_id#10399066, u_supplier_id#10399067, u_declaring_supplier_id#10399068, u_supplier_type#10399069, u_threshold#10399070, u_normalized_name#10399071, e_supplier_id#10399089, e_canonical_id#10399090, e_normalized_name#10399091, e_create_date#10399093]
: +- Filter (cast(pythonUDF0#10400694 as double) >= u_threshold#10399070)
: +- BatchEvalPython [calculate_similarity(u_normalized_name#10399071, e_normalized_name#10399091)#10399146], [pythonUDF0#10400694]
: +- Project [u_brand_context_id#10399066, u_supplier_id#10399067, u_declaring_supplier_id#10399068, u_supplier_type#10399069, u_threshold#10399070, u_normalized_name#10399071, e_supplier_id#10399089, e_canonical_id#10399090, e_normalized_name#10399091, e_create_date#10399093]
: +- BroadcastHashJoin [u_brand_context_id#10399066, u_location_key#10399075], [e_brand_context_id#10399088, e_location_key#10399092], Inner, BuildLeft, false
: :- BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, true], input[6, string, false]),false), [plan_id=356643]
: : +- Project [brand_context_id#10399276 AS u_brand_context_id#10399066, supplier_id#10399277 AS u_supplier_id#10399067, declaring_supplier_id#10399283 AS u_declaring_supplier_id#10399068, supplier_type#10399282 AS u_supplier_type#10399069, threshold#10399286 AS u_threshold#10399070, normalized_name#10395248 AS u_normalized_name#10399071, location_key#10395305 AS u_location_key#10399075]
: : +- SortMergeJoin [brand_context_id#10399276, supplier_id#10399277], [brand_context_id#10398214, supplier_id#10398215], LeftAnti
: : :- Sort [brand_context_id#10399276 ASC NULLS FIRST, supplier_id#10399277 ASC NULLS FIRST], false, 0
: : : +- Exchange hashpartitioning(brand_context_id#10399276, supplier_id#10399277, 10), ENSURE_REQUIREMENTS, [plan_id=356636]
: : : +- Project [brand_context_id#10399276, supplier_id#10399277, supplier_type#10399282, declaring_supplier_id#10399283, threshold#10399286, pythonUDF0#10400686 AS normalized_name#10395248, concat_ws(|, coalesce(pythonUDF1#10400687, ), coalesce(pythonUDF2#10400688, ), coalesce(pythonUDF3#10400689, )) AS location_key#10395305]
: : : +- BatchEvalPython [normalize_name(entity_name#10399278)#10395247, normalize_location(city#10399279)#10395260, normalize_location(state#10399280)#10395274, normalize_location(country#10399281)#10395289], [pythonUDF0#10400686, pythonUDF1#10400687, pythonUDF2#10400688, pythonUDF3#10400689]
: : : +- Project [brand_context_id#10399276, supplier_id#10399277, entity_name#10399278, city#10399279, state#10399280, country#10399281, supplier_type#10399282, declaring_supplier_id#10399283, threshold#10399286]
: : : +- Filter ((isnotnull(pythonUDF0#10400683) AND isnotnull(pythonUDF1#10400684)) AND isnotnull(pythonUDF2#10400685))
: : : +- BatchEvalPython [normalize_location(city#10399279)#10395260, normalize_location(state#10399280)#10395274, normalize_location(country#10399281)#10395289], [pythonUDF0#10400683, pythonUDF1#10400684, pythonUDF2#10400685]
: : : +- Filter (isnotnull(threshold#10399286) AND isnotnull(brand_context_id#10399276))
: : : +- InMemoryTableScan [brand_context_id#10399276, supplier_id#10399277, entity_name#10399278, city#10399279, state#10399280, country#10399281, supplier_type#10399282, declaring_supplier_id#10399283, threshold#10399286], [isnotnull(threshold#10399286), isnotnull(brand_context_id#10399276)]
: : : +- InMemoryRelation [brand_context_id#10399276, supplier_id#10399277, entity_name#10399278, city#10399279, state#10399280, country#10399281, supplier_type#10399282, declaring_supplier_id#10399283, threshold#10399286, create_date#10399287], StorageLevel(disk, memory, deserialized, 1 replicas)
: : : +- AdaptiveSparkPlan isFinalPlan=false
: : : +- Project [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, threshold#8983070, create_date#8983071]
: : : +- Filter isnull(canonical_id#8983068)
: : : +- InMemoryTableScan [brand_context_id#8983060, canonical_id#8983068, city#8983063, country#8983065, create_date#8983071, declaring_supplier_id#8983067, entity_name#8983062, state#8983064, supplier_id#8983061, supplier_type#8983066, threshold#8983070], [isnull(canonical_id#8983068)]
: : : +- InMemoryRelation [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, canonical_id#8983068, is_primary#8983069, threshold#8983070, create_date#8983071], StorageLevel(disk, memory, deserialized, 1 replicas)
: : : +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#8983060,supplier_id#8983061,entity_name#8983062,city#8983063,state#8983064,country#8983065,supplier_type#8983066,declaring_supplier_id#8983067,canonical_id#8983068,is_primary#8983069,threshold#8983070,create_date#8983071] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
: : +- Sort [brand_context_id#10398214 ASC NULLS FIRST, supplier_id#10398215 ASC NULLS FIRST], false, 0
: : +- Exchange hashpartitioning(brand_context_id#10398214, supplier_id#10398215, 10), ENSURE_REQUIREMENTS, [plan_id=356637]
: : +- Project [brand_context_id#10398214, supplier_id#10398215]
: : +- BroadcastHashJoin [brand_context_id#10398214, supplier_id#10398215], [brand_context_id#10399288, supplier_id#10399289], Inner, BuildLeft, false
: : :- BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, false], input[1, string, false]),false), [plan_id=356631]
: : : +- Filter (isnotnull(brand_context_id#10398214) AND isnotnull(supplier_id#10398215))
: : : +- InMemoryTableScan [brand_context_id#10398214, supplier_id#10398215], [isnotnull(brand_context_id#10398214), isnotnull(supplier_id#10398215)]
: : : +- InMemoryRelation [brand_context_id#10398214, supplier_id#10398215, entity_name#10398216, city#10398217, state#10398218, country#10398219, supplier_type#10398220, declaring_supplier_id#10398221, threshold#10398224, create_date#10398225], StorageLevel(disk, memory, deserialized, 1 replicas)
: : : +- AdaptiveSparkPlan isFinalPlan=false
: : : +- Project [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, threshold#8983070, create_date#8983071]
: : : +- Filter isnull(canonical_id#8983068)
: : : +- InMemoryTableScan [brand_context_id#8983060, canonical_id#8983068, city#8983063, country#8983065, create_date#8983071, declaring_supplier_id#8983067, entity_name#8983062, state#8983064, supplier_id#8983061, supplier_type#8983066, threshold#8983070], [isnull(canonical_id#8983068)]
: : : +- InMemoryRelation [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, canonical_id#8983068, is_primary#8983069, threshold#8983070, create_date#8983071], StorageLevel(disk, memory, deserialized, 1 replicas)
: : : +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#8983060,supplier_id#8983061,entity_name#8983062,city#8983063,state#8983064,country#8983065,supplier_type#8983066,declaring_supplier_id#8983067,canonical_id#8983068,is_primary#8983069,threshold#8983070,create_date#8983071] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
: : +- Project [brand_context_id#10399288, supplier_id#10399289]
: : +- Filter ((((isnotnull(is_primary#10399297) AND is_primary#10399297) AND isnotnull(canonical_id#10399296)) AND isnotnull(brand_context_id#10399288)) AND isnotnull(supplier_id#10399289))
: : +- InMemoryTableScan [brand_context_id#10399288, canonical_id#10399296, is_primary#10399297, supplier_id#10399289], [isnotnull(is_primary#10399297), is_primary#10399297, isnotnull(canonical_id#10399296), isnotnull(brand_context_id#10399288), isnotnull(supplier_id#10399289)]
: : +- InMemoryRelation [brand_context_id#10399288, supplier_id#10399289, canonical_id#10399296, is_primary#10399297, entity_name#10399290, city#10399291, state#10399292, country#10399293, threshold#10399298, create_date#10399299], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- AdaptiveSparkPlan isFinalPlan=false
: : +- Filter isnotnull(canonical_id#8983068)
: : +- InMemoryTableScan [brand_context_id#8983060, supplier_id#8983061, canonical_id#8983068, is_primary#8983069, entity_name#8983062, city#8983063, state#8983064, country#8983065, threshold#8983070, create_date#8983071], [isnotnull(canonical_id#8983068)]
: : +- InMemoryRelation [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, canonical_id#8983068, is_primary#8983069, threshold#8983070, create_date#8983071], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#8983060,supplier_id#8983061,entity_name#8983062,city#8983063,state#8983064,country#8983065,supplier_type#8983066,declaring_supplier_id#8983067,canonical_id#8983068,is_primary#8983069,threshold#8983070,create_date#8983071] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
: +- Project [brand_context_id#10399100 AS e_brand_context_id#10399088, supplier_id#10399101 AS e_supplier_id#10399089, canonical_id#10399108 AS e_canonical_id#10399090, pythonUDF0#10400690 AS e_normalized_name#10399091, concat_ws(|, coalesce(pythonUDF1#10400691, ), coalesce(pythonUDF2#10400692, ), coalesce(pythonUDF3#10400693, )) AS e_location_key#10399092, create_date#10399111 AS e_create_date#10399093]
: +- BatchEvalPython [normalize_name(entity_name#10399102)#10395321, normalize_location(city#10399103)#10395334, normalize_location(state#10399104)#10395348, normalize_location(country#10399105)#10395363], [pythonUDF0#10400690, pythonUDF1#10400691, pythonUDF2#10400692, pythonUDF3#10400693]
: +- Filter isnotnull(brand_context_id#10399100)
: +- InMemoryTableScan [brand_context_id#10399100, supplier_id#10399101, canonical_id#10399108, entity_name#10399102, city#10399103, state#10399104, country#10399105, create_date#10399111], [isnotnull(brand_context_id#10399100)]
: +- InMemoryRelation [brand_context_id#10399100, supplier_id#10399101, canonical_id#10399108, is_primary#10399109, entity_name#10399102, city#10399103, state#10399104, country#10399105, threshold#10399110, create_date#10399111], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- AdaptiveSparkPlan isFinalPlan=false
: +- Filter isnotnull(canonical_id#8983068)
: +- InMemoryTableScan [brand_context_id#8983060, supplier_id#8983061, canonical_id#8983068, is_primary#8983069, entity_name#8983062, city#8983063, state#8983064, country#8983065, threshold#8983070, create_date#8983071], [isnotnull(canonical_id#8983068)]
: +- InMemoryRelation [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, canonical_id#8983068, is_primary#8983069, threshold#8983070, create_date#8983071], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#8983060,supplier_id#8983061,entity_name#8983062,city#8983063,state#8983064,country#8983065,supplier_type#8983066,declaring_supplier_id#8983067,canonical_id#8983068,is_primary#8983069,threshold#8983070,create_date#8983071] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- Project [brand_context_id#10399312, supplier_id#10399313, declaring_supplier_id#10399319, supplier_type#10399318, supplier_id#10399313 AS canonical_id#10399260, 1 AS is_primary#10399261, threshold#10399322 AS configured_threshold#10399262, 0.0 AS match_percentage#10399263, 0.0 AS confidence_score#10399264, new_canonical AS match_reason#10399265]
+- SortMergeJoin [brand_context_id#10399312, supplier_id#10399313], [brand_context_id#10399187, supplier_id#10399188], LeftAnti
:- SortMergeJoin [brand_context_id#10399312, supplier_id#10399313], [brand_context_id#10399324, supplier_id#10399325], LeftAnti
: :- Sort [brand_context_id#10399312 ASC NULLS FIRST, supplier_id#10399313 ASC NULLS FIRST], false, 0
: : +- Exchange hashpartitioning(brand_context_id#10399312, supplier_id#10399313, 10), ENSURE_REQUIREMENTS, [plan_id=356666]
: : +- InMemoryTableScan [brand_context_id#10399312, supplier_id#10399313, supplier_type#10399318, declaring_supplier_id#10399319, threshold#10399322]
: : +- InMemoryRelation [brand_context_id#10399312, supplier_id#10399313, entity_name#10399314, city#10399315, state#10399316, country#10399317, supplier_type#10399318, declaring_supplier_id#10399319, threshold#10399322, create_date#10399323], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- AdaptiveSparkPlan isFinalPlan=false
: : +- Project [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, threshold#8983070, create_date#8983071]
: : +- Filter isnull(canonical_id#8983068)
: : +- InMemoryTableScan [brand_context_id#8983060, canonical_id#8983068, city#8983063, country#8983065, create_date#8983071, declaring_supplier_id#8983067, entity_name#8983062, state#8983064, supplier_id#8983061, supplier_type#8983066, threshold#8983070], [isnull(canonical_id#8983068)]
: : +- InMemoryRelation [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, canonical_id#8983068, is_primary#8983069, threshold#8983070, create_date#8983071], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#8983060,supplier_id#8983061,entity_name#8983062,city#8983063,state#8983064,country#8983065,supplier_type#8983066,declaring_supplier_id#8983067,canonical_id#8983068,is_primary#8983069,threshold#8983070,create_date#8983071] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
: +- Sort [brand_context_id#10399324 ASC NULLS FIRST, supplier_id#10399325 ASC NULLS FIRST], false, 0
: +- Exchange hashpartitioning(brand_context_id#10399324, supplier_id#10399325, 10), ENSURE_REQUIREMENTS, [plan_id=356667]
: +- Project [brand_context_id#10399324, supplier_id#10399325]
: +- BroadcastHashJoin [brand_context_id#10399324, supplier_id#10399325], [brand_context_id#10399336, supplier_id#10399337], Inner, BuildLeft, false
: :- BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, false], input[1, string, false]),false), [plan_id=356661]
: : +- Filter (isnotnull(brand_context_id#10399324) AND isnotnull(supplier_id#10399325))
: : +- InMemoryTableScan [brand_context_id#10399324, supplier_id#10399325], [isnotnull(brand_context_id#10399324), isnotnull(supplier_id#10399325)]
: : +- InMemoryRelation [brand_context_id#10399324, supplier_id#10399325, entity_name#10399326, city#10399327, state#10399328, country#10399329, supplier_type#10399330, declaring_supplier_id#10399331, threshold#10399334, create_date#10399335], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- AdaptiveSparkPlan isFinalPlan=false
: : +- Project [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, threshold#8983070, create_date#8983071]
: : +- Filter isnull(canonical_id#8983068)
: : +- InMemoryTableScan [brand_context_id#8983060, canonical_id#8983068, city#8983063, country#8983065, create_date#8983071, declaring_supplier_id#8983067, entity_name#8983062, state#8983064, supplier_id#8983061, supplier_type#8983066, threshold#8983070], [isnull(canonical_id#8983068)]
: : +- InMemoryRelation [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, canonical_id#8983068, is_primary#8983069, threshold#8983070, create_date#8983071], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#8983060,supplier_id#8983061,entity_name#8983062,city#8983063,state#8983064,country#8983065,supplier_type#8983066,declaring_supplier_id#8983067,canonical_id#8983068,is_primary#8983069,threshold#8983070,create_date#8983071] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
: +- Project [brand_context_id#10399336, supplier_id#10399337]
: +- Filter ((((isnotnull(is_primary#10399345) AND is_primary#10399345) AND isnotnull(canonical_id#10399344)) AND isnotnull(brand_context_id#10399336)) AND isnotnull(supplier_id#10399337))
: +- InMemoryTableScan [brand_context_id#10399336, canonical_id#10399344, is_primary#10399345, supplier_id#10399337], [isnotnull(is_primary#10399345), is_primary#10399345, isnotnull(canonical_id#10399344), isnotnull(brand_context_id#10399336), isnotnull(supplier_id#10399337)]
: +- InMemoryRelation [brand_context_id#10399336, supplier_id#10399337, canonical_id#10399344, is_primary#10399345, entity_name#10399338, city#10399339, state#10399340, country#10399341, threshold#10399346, create_date#10399347], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- AdaptiveSparkPlan isFinalPlan=false
: +- Filter isnotnull(canonical_id#8983068)
: +- InMemoryTableScan [brand_context_id#8983060, supplier_id#8983061, canonical_id#8983068, is_primary#8983069, entity_name#8983062, city#8983063, state#8983064, country#8983065, threshold#8983070, create_date#8983071], [isnotnull(canonical_id#8983068)]
: +- InMemoryRelation [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, canonical_id#8983068, is_primary#8983069, threshold#8983070, create_date#8983071], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#8983060,supplier_id#8983061,entity_name#8983062,city#8983063,state#8983064,country#8983065,supplier_type#8983066,declaring_supplier_id#8983067,canonical_id#8983068,is_primary#8983069,threshold#8983070,create_date#8983071] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- Project [u_brand_context_id#10399066 AS brand_context_id#10399187, u_supplier_id#10399067 AS supplier_id#10399188]
+- Filter (rank#10399167 = 1)
+- Window [row_number() windowspecdefinition(u_brand_context_id#10399066, u_supplier_id#10399067, similarity#10399147 DESC NULLS LAST, e_create_date#10399093 ASC NULLS FIRST, e_supplier_id#10399089 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank#10399167], [u_brand_context_id#10399066, u_supplier_id#10399067], [similarity#10399147 DESC NULLS LAST, e_create_date#10399093 ASC NULLS FIRST, e_supplier_id#10399089 ASC NULLS FIRST]
+- WindowGroupLimit [u_brand_context_id#10399066, u_supplier_id#10399067], [similarity#10399147 DESC NULLS LAST, e_create_date#10399093 ASC NULLS FIRST, e_supplier_id#10399089 ASC NULLS FIRST], row_number(), 1, Final
+- Sort [u_brand_context_id#10399066 ASC NULLS FIRST, u_supplier_id#10399067 ASC NULLS FIRST, similarity#10399147 DESC NULLS LAST, e_create_date#10399093 ASC NULLS FIRST, e_supplier_id#10399089 ASC NULLS FIRST], false, 0
+- Exchange hashpartitioning(u_brand_context_id#10399066, u_supplier_id#10399067, 10), ENSURE_REQUIREMENTS, [plan_id=356695]
+- WindowGroupLimit [u_brand_context_id#10399066, u_supplier_id#10399067], [similarity#10399147 DESC NULLS LAST, e_create_date#10399093 ASC NULLS FIRST, e_supplier_id#10399089 ASC NULLS FIRST], row_number(), 1, Partial
+- Sort [u_brand_context_id#10399066 ASC NULLS FIRST, u_supplier_id#10399067 ASC NULLS FIRST, similarity#10399147 DESC NULLS LAST, e_create_date#10399093 ASC NULLS FIRST, e_supplier_id#10399089 ASC NULLS FIRST], false, 0
+- Project [u_brand_context_id#10399066, u_supplier_id#10399067, e_supplier_id#10399089, e_create_date#10399093, pythonUDF0#10400708 AS similarity#10399147]
+- BatchEvalPython [calculate_similarity(u_normalized_name#10399071, e_normalized_name#10399091)#10399146], [pythonUDF0#10400708]
+- Project [u_brand_context_id#10399066, u_supplier_id#10399067, u_normalized_name#10399071, e_supplier_id#10399089, e_normalized_name#10399091, e_create_date#10399093]
+- Filter (cast(pythonUDF0#10400707 as double) >= u_threshold#10399070)
+- BatchEvalPython [calculate_similarity(u_normalized_name#10399071, e_normalized_name#10399091)#10399146], [pythonUDF0#10400707]
+- Project [u_brand_context_id#10399066, u_supplier_id#10399067, u_threshold#10399070, u_normalized_name#10399071, e_supplier_id#10399089, e_normalized_name#10399091, e_create_date#10399093]
+- BroadcastHashJoin [u_brand_context_id#10399066, u_location_key#10399075], [e_brand_context_id#10399088, e_location_key#10399092], Inner, BuildLeft, false
:- BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, true], input[4, string, false]),false), [plan_id=356683]
: +- Project [brand_context_id#10399209 AS u_brand_context_id#10399066, supplier_id#10399210 AS u_supplier_id#10399067, threshold#10399219 AS u_threshold#10399070, normalized_name#10395248 AS u_normalized_name#10399071, location_key#10395305 AS u_location_key#10399075]
: +- SortMergeJoin [brand_context_id#10399209, supplier_id#10399210], [brand_context_id#10399221, supplier_id#10399222], LeftAnti
: :- Sort [brand_context_id#10399209 ASC NULLS FIRST, supplier_id#10399210 ASC NULLS FIRST], false, 0
: : +- Exchange hashpartitioning(brand_context_id#10399209, supplier_id#10399210, 10), ENSURE_REQUIREMENTS, [plan_id=356676]
: : +- Project [brand_context_id#10399209, supplier_id#10399210, threshold#10399219, pythonUDF0#10400699 AS normalized_name#10395248, concat_ws(|, coalesce(pythonUDF1#10400700, ), coalesce(pythonUDF2#10400701, ), coalesce(pythonUDF3#10400702, )) AS location_key#10395305]
: : +- BatchEvalPython [normalize_name(entity_name#10399211)#10395247, normalize_location(city#10399212)#10395260, normalize_location(state#10399213)#10395274, normalize_location(country#10399214)#10395289], [pythonUDF0#10400699, pythonUDF1#10400700, pythonUDF2#10400701, pythonUDF3#10400702]
: : +- Project [brand_context_id#10399209, supplier_id#10399210, entity_name#10399211, city#10399212, state#10399213, country#10399214, threshold#10399219]
: : +- Filter ((isnotnull(pythonUDF0#10400696) AND isnotnull(pythonUDF1#10400697)) AND isnotnull(pythonUDF2#10400698))
: : +- BatchEvalPython [normalize_location(city#10399212)#10395260, normalize_location(state#10399213)#10395274, normalize_location(country#10399214)#10395289], [pythonUDF0#10400696, pythonUDF1#10400697, pythonUDF2#10400698]
: : +- Filter ((isnotnull(threshold#10399219) AND isnotnull(brand_context_id#10399209)) AND isnotnull(supplier_id#10399210))
: : +- InMemoryTableScan [brand_context_id#10399209, supplier_id#10399210, entity_name#10399211, city#10399212, state#10399213, country#10399214, threshold#10399219], [isnotnull(threshold#10399219), isnotnull(brand_context_id#10399209), isnotnull(supplier_id#10399210)]
: : +- InMemoryRelation [brand_context_id#10399209, supplier_id#10399210, entity_name#10399211, city#10399212, state#10399213, country#10399214, supplier_type#10399215, declaring_supplier_id#10399216, threshold#10399219, create_date#10399220], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- AdaptiveSparkPlan isFinalPlan=false
: : +- Project [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, threshold#8983070, create_date#8983071]
: : +- Filter isnull(canonical_id#8983068)
: : +- InMemoryTableScan [brand_context_id#8983060, canonical_id#8983068, city#8983063, country#8983065, create_date#8983071, declaring_supplier_id#8983067, entity_name#8983062, state#8983064, supplier_id#8983061, supplier_type#8983066, threshold#8983070], [isnull(canonical_id#8983068)]
: : +- InMemoryRelation [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, canonical_id#8983068, is_primary#8983069, threshold#8983070, create_date#8983071], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#8983060,supplier_id#8983061,entity_name#8983062,city#8983063,state#8983064,country#8983065,supplier_type#8983066,declaring_supplier_id#8983067,canonical_id#8983068,is_primary#8983069,threshold#8983070,create_date#8983071] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
: +- Sort [brand_context_id#10399221 ASC NULLS FIRST, supplier_id#10399222 ASC NULLS FIRST], false, 0
: +- Exchange hashpartitioning(brand_context_id#10399221, supplier_id#10399222, 10), ENSURE_REQUIREMENTS, [plan_id=356677]
: +- Project [brand_context_id#10399221, supplier_id#10399222]
: +- BroadcastHashJoin [brand_context_id#10399221, supplier_id#10399222], [brand_context_id#10399233, supplier_id#10399234], Inner, BuildLeft, false
: :- BroadcastExchange HashedRelationBroadcastMode(List(input[0, string, false], input[1, string, false]),false), [plan_id=356671]
: : +- Filter (isnotnull(brand_context_id#10399221) AND isnotnull(supplier_id#10399222))
: : +- InMemoryTableScan [brand_context_id#10399221, supplier_id#10399222], [isnotnull(brand_context_id#10399221), isnotnull(supplier_id#10399222)]
: : +- InMemoryRelation [brand_context_id#10399221, supplier_id#10399222, entity_name#10399223, city#10399224, state#10399225, country#10399226, supplier_type#10399227, declaring_supplier_id#10399228, threshold#10399231, create_date#10399232], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- AdaptiveSparkPlan isFinalPlan=false
: : +- Project [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, threshold#8983070, create_date#8983071]
: : +- Filter isnull(canonical_id#8983068)
: : +- InMemoryTableScan [brand_context_id#8983060, canonical_id#8983068, city#8983063, country#8983065, create_date#8983071, declaring_supplier_id#8983067, entity_name#8983062, state#8983064, supplier_id#8983061, supplier_type#8983066, threshold#8983070], [isnull(canonical_id#8983068)]
: : +- InMemoryRelation [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, canonical_id#8983068, is_primary#8983069, threshold#8983070, create_date#8983071], StorageLevel(disk, memory, deserialized, 1 replicas)
: : +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#8983060,supplier_id#8983061,entity_name#8983062,city#8983063,state#8983064,country#8983065,supplier_type#8983066,declaring_supplier_id#8983067,canonical_id#8983068,is_primary#8983069,threshold#8983070,create_date#8983071] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
: +- Project [brand_context_id#10399233, supplier_id#10399234]
: +- Filter ((((isnotnull(is_primary#10399242) AND is_primary#10399242) AND isnotnull(canonical_id#10399241)) AND isnotnull(brand_context_id#10399233)) AND isnotnull(supplier_id#10399234))
: +- InMemoryTableScan [brand_context_id#10399233, canonical_id#10399241, is_primary#10399242, supplier_id#10399234], [isnotnull(is_primary#10399242), is_primary#10399242, isnotnull(canonical_id#10399241), isnotnull(brand_context_id#10399233), isnotnull(supplier_id#10399234)]
: +- InMemoryRelation [brand_context_id#10399233, supplier_id#10399234, canonical_id#10399241, is_primary#10399242, entity_name#10399235, city#10399236, state#10399237, country#10399238, threshold#10399243, create_date#10399244], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- AdaptiveSparkPlan isFinalPlan=false
: +- Filter isnotnull(canonical_id#8983068)
: +- InMemoryTableScan [brand_context_id#8983060, supplier_id#8983061, canonical_id#8983068, is_primary#8983069, entity_name#8983062, city#8983063, state#8983064, country#8983065, threshold#8983070, create_date#8983071], [isnotnull(canonical_id#8983068)]
: +- InMemoryRelation [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, canonical_id#8983068, is_primary#8983069, threshold#8983070, create_date#8983071], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#8983060,supplier_id#8983061,entity_name#8983062,city#8983063,state#8983064,country#8983065,supplier_type#8983066,declaring_supplier_id#8983067,canonical_id#8983068,is_primary#8983069,threshold#8983070,create_date#8983071] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- Project [brand_context_id#10399348 AS e_brand_context_id#10399088, supplier_id#10399349 AS e_supplier_id#10399089, pythonUDF0#10400703 AS e_normalized_name#10399091, concat_ws(|, coalesce(pythonUDF1#10400704, ), coalesce(pythonUDF2#10400705, ), coalesce(pythonUDF3#10400706, )) AS e_location_key#10399092, create_date#10399359 AS e_create_date#10399093]
+- BatchEvalPython [normalize_name(entity_name#10399350)#10395321, normalize_location(city#10399351)#10395334, normalize_location(state#10399352)#10395348, normalize_location(country#10399353)#10395363], [pythonUDF0#10400703, pythonUDF1#10400704, pythonUDF2#10400705, pythonUDF3#10400706]
+- Filter isnotnull(brand_context_id#10399348)
+- InMemoryTableScan [brand_context_id#10399348, supplier_id#10399349, entity_name#10399350, city#10399351, state#10399352, country#10399353, create_date#10399359], [isnotnull(brand_context_id#10399348)]
+- InMemoryRelation [brand_context_id#10399348, supplier_id#10399349, canonical_id#10399356, is_primary#10399357, entity_name#10399350, city#10399351, state#10399352, country#10399353, threshold#10399358, create_date#10399359], StorageLevel(disk, memory, deserialized, 1 replicas)
+- AdaptiveSparkPlan isFinalPlan=false
+- Filter isnotnull(canonical_id#8983068)
+- InMemoryTableScan [brand_context_id#8983060, supplier_id#8983061, canonical_id#8983068, is_primary#8983069, entity_name#8983062, city#8983063, state#8983064, country#8983065, threshold#8983070, create_date#8983071], [isnotnull(canonical_id#8983068)]
+- InMemoryRelation [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, canonical_id#8983068, is_primary#8983069, threshold#8983070, create_date#8983071], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#8983060,supplier_id#8983061,entity_name#8983062,city#8983063,state#8983064,country#8983065,supplier_type#8983066,declaring_supplier_id#8983067,canonical_id#8983068,is_primary#8983069,threshold#8983070,create_date#8983071] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(3) InMemoryTableScan
Output [5]: [brand_context_id#8983060, supplier_id#8983061, supplier_type#8983066, declaring_supplier_id#8983067, threshold#8983070]
Arguments: [brand_context_id#8983060, supplier_id#8983061, supplier_type#8983066, declaring_supplier_id#8983067, threshold#8983070], [isnotnull(brand_context_id#8983060), isnotnull(supplier_id#8983061)]
(4) InMemoryRelation
Arguments: [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, threshold#8983070, create_date#8983071], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@7795a5ad,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Project [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, threshold#8983070, create_date#8983071]
+- *(1) Filter isnull(canonical_id#8983068)
+- TableCacheQueryStage 0
+- InMemoryTableScan [brand_context_id#8983060, canonical_id#8983068, city#8983063, country#8983065, create_date#8983071, declaring_supplier_id#8983067, entity_name#8983062, state#8983064, supplier_id#8983061, supplier_type#8983066, threshold#8983070], [isnull(canonical_id#8983068)]
+- InMemoryRelation [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, canonical_id#8983068, is_primary#8983069, threshold#8983070, create_date#8983071], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#8983060,supplier_id#8983061,entity_name#8983062,city#8983063,state#8983064,country#8983065,supplier_type#8983066,declaring_supplier_id#8983067,canonical_id#8983068,is_primary#8983069,threshold#8983070,create_date#8983071] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Project [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, threshold#8983070, create_date#8983071]
+- Filter isnull(canonical_id#8983068)
+- InMemoryTableScan [brand_context_id#8983060, canonical_id#8983068, city#8983063, country#8983065, create_date#8983071, declaring_supplier_id#8983067, entity_name#8983062, state#8983064, supplier_id#8983061, supplier_type#8983066, threshold#8983070], [isnull(canonical_id#8983068)]
+- InMemoryRelation [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, canonical_id#8983068, is_primary#8983069, threshold#8983070, create_date#8983071], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#8983060,supplier_id#8983061,entity_name#8983062,city#8983063,state#8983064,country#8983065,supplier_type#8983066,declaring_supplier_id#8983067,canonical_id#8983068,is_primary#8983069,threshold#8983070,create_date#8983071] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(5) InMemoryTableScan
Output [11]: [brand_context_id#8983060, canonical_id#8983068, city#8983063, country#8983065, create_date#8983071, declaring_supplier_id#8983067, entity_name#8983062, state#8983064, supplier_id#8983061, supplier_type#8983066, threshold#8983070]
Arguments: [brand_context_id#8983060, canonical_id#8983068, city#8983063, country#8983065, create_date#8983071, declaring_supplier_id#8983067, entity_name#8983062, state#8983064, supplier_id#8983061, supplier_type#8983066, threshold#8983070], [isnull(canonical_id#8983068)]
(6) InMemoryRelation
Arguments: [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, canonical_id#8983068, is_primary#8983069, threshold#8983070, create_date#8983071], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@7795a5ad,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#8983060,supplier_id#8983061,entity_name#8983062,city#8983063,state#8983064,country#8983065,supplier_type#8983066,declaring_supplier_id#8983067,canonical_id#8983068,is_primary#8983069,threshold#8983070,create_date#8983071] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(7) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [codegen id : 1]
Output [12]: [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, canonical_id#8983068, is_primary#8983069, threshold#8983070, create_date#8983071]
ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,country:string,supplier_type:string,declaring_supplier_id:string,canonical_id:string,is_primary:boolean,threshold:double,create_date:date>
(8) Filter
Input [11]: [brand_context_id#8983060, canonical_id#8983068, city#8983063, country#8983065, create_date#8983071, declaring_supplier_id#8983067, entity_name#8983062, state#8983064, supplier_id#8983061, supplier_type#8983066, threshold#8983070]
Condition : isnull(canonical_id#8983068)
(9) Project
Output [10]: [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, threshold#8983070, create_date#8983071]
Input [11]: [brand_context_id#8983060, canonical_id#8983068, city#8983063, country#8983065, create_date#8983071, declaring_supplier_id#8983067, entity_name#8983062, state#8983064, supplier_id#8983061, supplier_type#8983066, threshold#8983070]
(10) AdaptiveSparkPlan
Output [10]: [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, threshold#8983070, create_date#8983071]
Arguments: isFinalPlan=false
(11) Filter
Input [5]: [brand_context_id#8983060, supplier_id#8983061, supplier_type#8983066, declaring_supplier_id#8983067, threshold#8983070]
Condition : (isnotnull(brand_context_id#8983060) AND isnotnull(supplier_id#8983061))
(12) BroadcastExchange
Input [5]: [brand_context_id#8983060, supplier_id#8983061, supplier_type#8983066, declaring_supplier_id#8983067, threshold#8983070]
Arguments: HashedRelationBroadcastMode(List(input[0, string, false], input[1, string, false]),false), [plan_id=361541]
(13) InMemoryTableScan
Output [4]: [brand_context_id#10397728, canonical_id#10397736, is_primary#10397737, supplier_id#10397729]
Arguments: [brand_context_id#10397728, canonical_id#10397736, is_primary#10397737, supplier_id#10397729], [isnotnull(is_primary#10397737), is_primary#10397737, isnotnull(canonical_id#10397736), isnotnull(brand_context_id#10397728), isnotnull(supplier_id#10397729)]
(14) InMemoryRelation
Arguments: [brand_context_id#10397728, supplier_id#10397729, canonical_id#10397736, is_primary#10397737, entity_name#10397730, city#10397731, state#10397732, country#10397733, threshold#10397738, create_date#10397739], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@7795a5ad,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Filter isnotnull(canonical_id#8983068)
+- TableCacheQueryStage 0
+- InMemoryTableScan [brand_context_id#8983060, supplier_id#8983061, canonical_id#8983068, is_primary#8983069, entity_name#8983062, city#8983063, state#8983064, country#8983065, threshold#8983070, create_date#8983071], [isnotnull(canonical_id#8983068)]
+- InMemoryRelation [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, canonical_id#8983068, is_primary#8983069, threshold#8983070, create_date#8983071], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#8983060,supplier_id#8983061,entity_name#8983062,city#8983063,state#8983064,country#8983065,supplier_type#8983066,declaring_supplier_id#8983067,canonical_id#8983068,is_primary#8983069,threshold#8983070,create_date#8983071] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Filter isnotnull(canonical_id#8983068)
+- InMemoryTableScan [brand_context_id#8983060, supplier_id#8983061, canonical_id#8983068, is_primary#8983069, entity_name#8983062, city#8983063, state#8983064, country#8983065, threshold#8983070, create_date#8983071], [isnotnull(canonical_id#8983068)]
+- InMemoryRelation [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, canonical_id#8983068, is_primary#8983069, threshold#8983070, create_date#8983071], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#8983060,supplier_id#8983061,entity_name#8983062,city#8983063,state#8983064,country#8983065,supplier_type#8983066,declaring_supplier_id#8983067,canonical_id#8983068,is_primary#8983069,threshold#8983070,create_date#8983071] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(15) InMemoryTableScan
Output [10]: [brand_context_id#8983060, supplier_id#8983061, canonical_id#8983068, is_primary#8983069, entity_name#8983062, city#8983063, state#8983064, country#8983065, threshold#8983070, create_date#8983071]
Arguments: [brand_context_id#8983060, supplier_id#8983061, canonical_id#8983068, is_primary#8983069, entity_name#8983062, city#8983063, state#8983064, country#8983065, threshold#8983070, create_date#8983071], [isnotnull(canonical_id#8983068)]
(16) InMemoryRelation
Arguments: [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, canonical_id#8983068, is_primary#8983069, threshold#8983070, create_date#8983071], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@7795a5ad,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#8983060,supplier_id#8983061,entity_name#8983062,city#8983063,state#8983064,country#8983065,supplier_type#8983066,declaring_supplier_id#8983067,canonical_id#8983068,is_primary#8983069,threshold#8983070,create_date#8983071] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(17) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [codegen id : 1]
Output [12]: [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, canonical_id#8983068, is_primary#8983069, threshold#8983070, create_date#8983071]
ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,country:string,supplier_type:string,declaring_supplier_id:string,canonical_id:string,is_primary:boolean,threshold:double,create_date:date>
(18) Filter
Input [10]: [brand_context_id#8983060, supplier_id#8983061, canonical_id#8983068, is_primary#8983069, entity_name#8983062, city#8983063, state#8983064, country#8983065, threshold#8983070, create_date#8983071]
Condition : isnotnull(canonical_id#8983068)
(19) AdaptiveSparkPlan
Output [10]: [brand_context_id#8983060, supplier_id#8983061, canonical_id#8983068, is_primary#8983069, entity_name#8983062, city#8983063, state#8983064, country#8983065, threshold#8983070, create_date#8983071]
Arguments: isFinalPlan=false
(20) Filter
Input [4]: [brand_context_id#10397728, canonical_id#10397736, is_primary#10397737, supplier_id#10397729]
Condition : ((((isnotnull(is_primary#10397737) AND is_primary#10397737) AND isnotnull(canonical_id#10397736)) AND isnotnull(brand_context_id#10397728)) AND isnotnull(supplier_id#10397729))
(21) Project
Output [3]: [brand_context_id#10397728, supplier_id#10397729, canonical_id#10397736]
Input [4]: [brand_context_id#10397728, canonical_id#10397736, is_primary#10397737, supplier_id#10397729]
(22) BroadcastHashJoin
Left keys [2]: [brand_context_id#8983060, supplier_id#8983061]
Right keys [2]: [brand_context_id#10397728, supplier_id#10397729]
Join type: Inner
Join condition: None
(23) Project
Output [10]: [brand_context_id#8983060, supplier_id#8983061, declaring_supplier_id#8983067, supplier_type#8983066, canonical_id#10397736, 0 AS is_primary#10397910, threshold#8983070 AS configured_threshold#10397911, 100.0 AS match_percentage#10397912, 1.0 AS confidence_score#10397913, exact_id_match AS match_reason#10397914]
Input [8]: [brand_context_id#8983060, supplier_id#8983061, supplier_type#8983066, declaring_supplier_id#8983067, threshold#8983070, brand_context_id#10397728, supplier_id#10397729, canonical_id#10397736]
(24) InMemoryTableScan
Output [9]: [brand_context_id#10399276, supplier_id#10399277, entity_name#10399278, city#10399279, state#10399280, country#10399281, supplier_type#10399282, declaring_supplier_id#10399283, threshold#10399286]
Arguments: [brand_context_id#10399276, supplier_id#10399277, entity_name#10399278, city#10399279, state#10399280, country#10399281, supplier_type#10399282, declaring_supplier_id#10399283, threshold#10399286], [isnotnull(threshold#10399286), isnotnull(brand_context_id#10399276)]
(25) InMemoryRelation
Arguments: [brand_context_id#10399276, supplier_id#10399277, entity_name#10399278, city#10399279, state#10399280, country#10399281, supplier_type#10399282, declaring_supplier_id#10399283, threshold#10399286, create_date#10399287], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@7795a5ad,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Project [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, threshold#8983070, create_date#8983071]
+- *(1) Filter isnull(canonical_id#8983068)
+- TableCacheQueryStage 0
+- InMemoryTableScan [brand_context_id#8983060, canonical_id#8983068, city#8983063, country#8983065, create_date#8983071, declaring_supplier_id#8983067, entity_name#8983062, state#8983064, supplier_id#8983061, supplier_type#8983066, threshold#8983070], [isnull(canonical_id#8983068)]
+- InMemoryRelation [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, canonical_id#8983068, is_primary#8983069, threshold#8983070, create_date#8983071], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#8983060,supplier_id#8983061,entity_name#8983062,city#8983063,state#8983064,country#8983065,supplier_type#8983066,declaring_supplier_id#8983067,canonical_id#8983068,is_primary#8983069,threshold#8983070,create_date#8983071] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Project [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, threshold#8983070, create_date#8983071]
+- Filter isnull(canonical_id#8983068)
+- InMemoryTableScan [brand_context_id#8983060, canonical_id#8983068, city#8983063, country#8983065, create_date#8983071, declaring_supplier_id#8983067, entity_name#8983062, state#8983064, supplier_id#8983061, supplier_type#8983066, threshold#8983070], [isnull(canonical_id#8983068)]
+- InMemoryRelation [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, canonical_id#8983068, is_primary#8983069, threshold#8983070, create_date#8983071], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#8983060,supplier_id#8983061,entity_name#8983062,city#8983063,state#8983064,country#8983065,supplier_type#8983066,declaring_supplier_id#8983067,canonical_id#8983068,is_primary#8983069,threshold#8983070,create_date#8983071] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(26) AdaptiveSparkPlan
Output [10]: [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, threshold#8983070, create_date#8983071]
Arguments: isFinalPlan=false
(27) Filter
Input [9]: [brand_context_id#10399276, supplier_id#10399277, entity_name#10399278, city#10399279, state#10399280, country#10399281, supplier_type#10399282, declaring_supplier_id#10399283, threshold#10399286]
Condition : (isnotnull(threshold#10399286) AND isnotnull(brand_context_id#10399276))
(28) BatchEvalPython
Input [9]: [brand_context_id#10399276, supplier_id#10399277, entity_name#10399278, city#10399279, state#10399280, country#10399281, supplier_type#10399282, declaring_supplier_id#10399283, threshold#10399286]
Arguments: [normalize_location(city#10399279)#10395260, normalize_location(state#10399280)#10395274, normalize_location(country#10399281)#10395289], [pythonUDF0#10400683, pythonUDF1#10400684, pythonUDF2#10400685]
(29) Filter
Input [12]: [brand_context_id#10399276, supplier_id#10399277, entity_name#10399278, city#10399279, state#10399280, country#10399281, supplier_type#10399282, declaring_supplier_id#10399283, threshold#10399286, pythonUDF0#10400683, pythonUDF1#10400684, pythonUDF2#10400685]
Condition : ((isnotnull(pythonUDF0#10400683) AND isnotnull(pythonUDF1#10400684)) AND isnotnull(pythonUDF2#10400685))
(30) Project
Output [9]: [brand_context_id#10399276, supplier_id#10399277, entity_name#10399278, city#10399279, state#10399280, country#10399281, supplier_type#10399282, declaring_supplier_id#10399283, threshold#10399286]
Input [12]: [brand_context_id#10399276, supplier_id#10399277, entity_name#10399278, city#10399279, state#10399280, country#10399281, supplier_type#10399282, declaring_supplier_id#10399283, threshold#10399286, pythonUDF0#10400683, pythonUDF1#10400684, pythonUDF2#10400685]
(31) BatchEvalPython
Input [9]: [brand_context_id#10399276, supplier_id#10399277, entity_name#10399278, city#10399279, state#10399280, country#10399281, supplier_type#10399282, declaring_supplier_id#10399283, threshold#10399286]
Arguments: [normalize_name(entity_name#10399278)#10395247, normalize_location(city#10399279)#10395260, normalize_location(state#10399280)#10395274, normalize_location(country#10399281)#10395289], [pythonUDF0#10400686, pythonUDF1#10400687, pythonUDF2#10400688, pythonUDF3#10400689]
(32) Project
Output [7]: [brand_context_id#10399276, supplier_id#10399277, supplier_type#10399282, declaring_supplier_id#10399283, threshold#10399286, pythonUDF0#10400686 AS normalized_name#10395248, concat_ws(|, coalesce(pythonUDF1#10400687, ), coalesce(pythonUDF2#10400688, ), coalesce(pythonUDF3#10400689, )) AS location_key#10395305]
Input [13]: [brand_context_id#10399276, supplier_id#10399277, entity_name#10399278, city#10399279, state#10399280, country#10399281, supplier_type#10399282, declaring_supplier_id#10399283, threshold#10399286, pythonUDF0#10400686, pythonUDF1#10400687, pythonUDF2#10400688, pythonUDF3#10400689]
(33) Exchange
Input [7]: [brand_context_id#10399276, supplier_id#10399277, supplier_type#10399282, declaring_supplier_id#10399283, threshold#10399286, normalized_name#10395248, location_key#10395305]
Arguments: hashpartitioning(brand_context_id#10399276, supplier_id#10399277, 10), ENSURE_REQUIREMENTS, [plan_id=361549]
(34) Sort
Input [7]: [brand_context_id#10399276, supplier_id#10399277, supplier_type#10399282, declaring_supplier_id#10399283, threshold#10399286, normalized_name#10395248, location_key#10395305]
Arguments: [brand_context_id#10399276 ASC NULLS FIRST, supplier_id#10399277 ASC NULLS FIRST], false, 0
(35) InMemoryTableScan
Output [2]: [brand_context_id#10398214, supplier_id#10398215]
Arguments: [brand_context_id#10398214, supplier_id#10398215], [isnotnull(brand_context_id#10398214), isnotnull(supplier_id#10398215)]
(36) InMemoryRelation
Arguments: [brand_context_id#10398214, supplier_id#10398215, entity_name#10398216, city#10398217, state#10398218, country#10398219, supplier_type#10398220, declaring_supplier_id#10398221, threshold#10398224, create_date#10398225], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@7795a5ad,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Project [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, threshold#8983070, create_date#8983071]
+- *(1) Filter isnull(canonical_id#8983068)
+- TableCacheQueryStage 0
+- InMemoryTableScan [brand_context_id#8983060, canonical_id#8983068, city#8983063, country#8983065, create_date#8983071, declaring_supplier_id#8983067, entity_name#8983062, state#8983064, supplier_id#8983061, supplier_type#8983066, threshold#8983070], [isnull(canonical_id#8983068)]
+- InMemoryRelation [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, canonical_id#8983068, is_primary#8983069, threshold#8983070, create_date#8983071], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#8983060,supplier_id#8983061,entity_name#8983062,city#8983063,state#8983064,country#8983065,supplier_type#8983066,declaring_supplier_id#8983067,canonical_id#8983068,is_primary#8983069,threshold#8983070,create_date#8983071] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Project [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, threshold#8983070, create_date#8983071]
+- Filter isnull(canonical_id#8983068)
+- InMemoryTableScan [brand_context_id#8983060, canonical_id#8983068, city#8983063, country#8983065, create_date#8983071, declaring_supplier_id#8983067, entity_name#8983062, state#8983064, supplier_id#8983061, supplier_type#8983066, threshold#8983070], [isnull(canonical_id#8983068)]
+- InMemoryRelation [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, canonical_id#8983068, is_primary#8983069, threshold#8983070, create_date#8983071], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#8983060,supplier_id#8983061,entity_name#8983062,city#8983063,state#8983064,country#8983065,supplier_type#8983066,declaring_supplier_id#8983067,canonical_id#8983068,is_primary#8983069,threshold#8983070,create_date#8983071] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(37) AdaptiveSparkPlan
Output [10]: [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, threshold#8983070, create_date#8983071]
Arguments: isFinalPlan=false
(38) Filter
Input [2]: [brand_context_id#10398214, supplier_id#10398215]
Condition : (isnotnull(brand_context_id#10398214) AND isnotnull(supplier_id#10398215))
(39) BroadcastExchange
Input [2]: [brand_context_id#10398214, supplier_id#10398215]
Arguments: HashedRelationBroadcastMode(List(input[0, string, false], input[1, string, false]),false), [plan_id=361544]
(40) InMemoryTableScan
Output [4]: [brand_context_id#10399288, canonical_id#10399296, is_primary#10399297, supplier_id#10399289]
Arguments: [brand_context_id#10399288, canonical_id#10399296, is_primary#10399297, supplier_id#10399289], [isnotnull(is_primary#10399297), is_primary#10399297, isnotnull(canonical_id#10399296), isnotnull(brand_context_id#10399288), isnotnull(supplier_id#10399289)]
(41) InMemoryRelation
Arguments: [brand_context_id#10399288, supplier_id#10399289, canonical_id#10399296, is_primary#10399297, entity_name#10399290, city#10399291, state#10399292, country#10399293, threshold#10399298, create_date#10399299], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@7795a5ad,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Filter isnotnull(canonical_id#8983068)
+- TableCacheQueryStage 0
+- InMemoryTableScan [brand_context_id#8983060, supplier_id#8983061, canonical_id#8983068, is_primary#8983069, entity_name#8983062, city#8983063, state#8983064, country#8983065, threshold#8983070, create_date#8983071], [isnotnull(canonical_id#8983068)]
+- InMemoryRelation [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, canonical_id#8983068, is_primary#8983069, threshold#8983070, create_date#8983071], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#8983060,supplier_id#8983061,entity_name#8983062,city#8983063,state#8983064,country#8983065,supplier_type#8983066,declaring_supplier_id#8983067,canonical_id#8983068,is_primary#8983069,threshold#8983070,create_date#8983071] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Filter isnotnull(canonical_id#8983068)
+- InMemoryTableScan [brand_context_id#8983060, supplier_id#8983061, canonical_id#8983068, is_primary#8983069, entity_name#8983062, city#8983063, state#8983064, country#8983065, threshold#8983070, create_date#8983071], [isnotnull(canonical_id#8983068)]
+- InMemoryRelation [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, canonical_id#8983068, is_primary#8983069, threshold#8983070, create_date#8983071], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#8983060,supplier_id#8983061,entity_name#8983062,city#8983063,state#8983064,country#8983065,supplier_type#8983066,declaring_supplier_id#8983067,canonical_id#8983068,is_primary#8983069,threshold#8983070,create_date#8983071] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(42) AdaptiveSparkPlan
Output [10]: [brand_context_id#8983060, supplier_id#8983061, canonical_id#8983068, is_primary#8983069, entity_name#8983062, city#8983063, state#8983064, country#8983065, threshold#8983070, create_date#8983071]
Arguments: isFinalPlan=false
(43) Filter
Input [4]: [brand_context_id#10399288, canonical_id#10399296, is_primary#10399297, supplier_id#10399289]
Condition : ((((isnotnull(is_primary#10399297) AND is_primary#10399297) AND isnotnull(canonical_id#10399296)) AND isnotnull(brand_context_id#10399288)) AND isnotnull(supplier_id#10399289))
(44) Project
Output [2]: [brand_context_id#10399288, supplier_id#10399289]
Input [4]: [brand_context_id#10399288, canonical_id#10399296, is_primary#10399297, supplier_id#10399289]
(45) BroadcastHashJoin
Left keys [2]: [brand_context_id#10398214, supplier_id#10398215]
Right keys [2]: [brand_context_id#10399288, supplier_id#10399289]
Join type: Inner
Join condition: None
(46) Project
Output [2]: [brand_context_id#10398214, supplier_id#10398215]
Input [4]: [brand_context_id#10398214, supplier_id#10398215, brand_context_id#10399288, supplier_id#10399289]
(47) Exchange
Input [2]: [brand_context_id#10398214, supplier_id#10398215]
Arguments: hashpartitioning(brand_context_id#10398214, supplier_id#10398215, 10), ENSURE_REQUIREMENTS, [plan_id=361550]
(48) Sort
Input [2]: [brand_context_id#10398214, supplier_id#10398215]
Arguments: [brand_context_id#10398214 ASC NULLS FIRST, supplier_id#10398215 ASC NULLS FIRST], false, 0
(49) SortMergeJoin
Left keys [2]: [brand_context_id#10399276, supplier_id#10399277]
Right keys [2]: [brand_context_id#10398214, supplier_id#10398215]
Join type: LeftAnti
Join condition: None
(50) Project
Output [7]: [brand_context_id#10399276 AS u_brand_context_id#10399066, supplier_id#10399277 AS u_supplier_id#10399067, declaring_supplier_id#10399283 AS u_declaring_supplier_id#10399068, supplier_type#10399282 AS u_supplier_type#10399069, threshold#10399286 AS u_threshold#10399070, normalized_name#10395248 AS u_normalized_name#10399071, location_key#10395305 AS u_location_key#10399075]
Input [7]: [brand_context_id#10399276, supplier_id#10399277, supplier_type#10399282, declaring_supplier_id#10399283, threshold#10399286, normalized_name#10395248, location_key#10395305]
(51) BroadcastExchange
Input [7]: [u_brand_context_id#10399066, u_supplier_id#10399067, u_declaring_supplier_id#10399068, u_supplier_type#10399069, u_threshold#10399070, u_normalized_name#10399071, u_location_key#10399075]
Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[6, string, false]),false), [plan_id=361556]
(52) InMemoryTableScan
Output [8]: [brand_context_id#10399100, supplier_id#10399101, canonical_id#10399108, entity_name#10399102, city#10399103, state#10399104, country#10399105, create_date#10399111]
Arguments: [brand_context_id#10399100, supplier_id#10399101, canonical_id#10399108, entity_name#10399102, city#10399103, state#10399104, country#10399105, create_date#10399111], [isnotnull(brand_context_id#10399100)]
(53) InMemoryRelation
Arguments: [brand_context_id#10399100, supplier_id#10399101, canonical_id#10399108, is_primary#10399109, entity_name#10399102, city#10399103, state#10399104, country#10399105, threshold#10399110, create_date#10399111], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@7795a5ad,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Filter isnotnull(canonical_id#8983068)
+- TableCacheQueryStage 0
+- InMemoryTableScan [brand_context_id#8983060, supplier_id#8983061, canonical_id#8983068, is_primary#8983069, entity_name#8983062, city#8983063, state#8983064, country#8983065, threshold#8983070, create_date#8983071], [isnotnull(canonical_id#8983068)]
+- InMemoryRelation [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, canonical_id#8983068, is_primary#8983069, threshold#8983070, create_date#8983071], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#8983060,supplier_id#8983061,entity_name#8983062,city#8983063,state#8983064,country#8983065,supplier_type#8983066,declaring_supplier_id#8983067,canonical_id#8983068,is_primary#8983069,threshold#8983070,create_date#8983071] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Filter isnotnull(canonical_id#8983068)
+- InMemoryTableScan [brand_context_id#8983060, supplier_id#8983061, canonical_id#8983068, is_primary#8983069, entity_name#8983062, city#8983063, state#8983064, country#8983065, threshold#8983070, create_date#8983071], [isnotnull(canonical_id#8983068)]
+- InMemoryRelation [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, canonical_id#8983068, is_primary#8983069, threshold#8983070, create_date#8983071], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#8983060,supplier_id#8983061,entity_name#8983062,city#8983063,state#8983064,country#8983065,supplier_type#8983066,declaring_supplier_id#8983067,canonical_id#8983068,is_primary#8983069,threshold#8983070,create_date#8983071] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(54) AdaptiveSparkPlan
Output [10]: [brand_context_id#8983060, supplier_id#8983061, canonical_id#8983068, is_primary#8983069, entity_name#8983062, city#8983063, state#8983064, country#8983065, threshold#8983070, create_date#8983071]
Arguments: isFinalPlan=false
(55) Filter
Input [8]: [brand_context_id#10399100, supplier_id#10399101, canonical_id#10399108, entity_name#10399102, city#10399103, state#10399104, country#10399105, create_date#10399111]
Condition : isnotnull(brand_context_id#10399100)
(56) BatchEvalPython
Input [8]: [brand_context_id#10399100, supplier_id#10399101, canonical_id#10399108, entity_name#10399102, city#10399103, state#10399104, country#10399105, create_date#10399111]
Arguments: [normalize_name(entity_name#10399102)#10395321, normalize_location(city#10399103)#10395334, normalize_location(state#10399104)#10395348, normalize_location(country#10399105)#10395363], [pythonUDF0#10400690, pythonUDF1#10400691, pythonUDF2#10400692, pythonUDF3#10400693]
(57) Project
Output [6]: [brand_context_id#10399100 AS e_brand_context_id#10399088, supplier_id#10399101 AS e_supplier_id#10399089, canonical_id#10399108 AS e_canonical_id#10399090, pythonUDF0#10400690 AS e_normalized_name#10399091, concat_ws(|, coalesce(pythonUDF1#10400691, ), coalesce(pythonUDF2#10400692, ), coalesce(pythonUDF3#10400693, )) AS e_location_key#10399092, create_date#10399111 AS e_create_date#10399093]
Input [12]: [brand_context_id#10399100, supplier_id#10399101, canonical_id#10399108, entity_name#10399102, city#10399103, state#10399104, country#10399105, create_date#10399111, pythonUDF0#10400690, pythonUDF1#10400691, pythonUDF2#10400692, pythonUDF3#10400693]
(58) BroadcastHashJoin
Left keys [2]: [u_brand_context_id#10399066, u_location_key#10399075]
Right keys [2]: [e_brand_context_id#10399088, e_location_key#10399092]
Join type: Inner
Join condition: None
(59) Project
Output [10]: [u_brand_context_id#10399066, u_supplier_id#10399067, u_declaring_supplier_id#10399068, u_supplier_type#10399069, u_threshold#10399070, u_normalized_name#10399071, e_supplier_id#10399089, e_canonical_id#10399090, e_normalized_name#10399091, e_create_date#10399093]
Input [13]: [u_brand_context_id#10399066, u_supplier_id#10399067, u_declaring_supplier_id#10399068, u_supplier_type#10399069, u_threshold#10399070, u_normalized_name#10399071, u_location_key#10399075, e_brand_context_id#10399088, e_supplier_id#10399089, e_canonical_id#10399090, e_normalized_name#10399091, e_location_key#10399092, e_create_date#10399093]
(60) BatchEvalPython
Input [10]: [u_brand_context_id#10399066, u_supplier_id#10399067, u_declaring_supplier_id#10399068, u_supplier_type#10399069, u_threshold#10399070, u_normalized_name#10399071, e_supplier_id#10399089, e_canonical_id#10399090, e_normalized_name#10399091, e_create_date#10399093]
Arguments: [calculate_similarity(u_normalized_name#10399071, e_normalized_name#10399091)#10399146], [pythonUDF0#10400694]
(61) Filter
Input [11]: [u_brand_context_id#10399066, u_supplier_id#10399067, u_declaring_supplier_id#10399068, u_supplier_type#10399069, u_threshold#10399070, u_normalized_name#10399071, e_supplier_id#10399089, e_canonical_id#10399090, e_normalized_name#10399091, e_create_date#10399093, pythonUDF0#10400694]
Condition : (cast(pythonUDF0#10400694 as double) >= u_threshold#10399070)
(62) Project
Output [10]: [u_brand_context_id#10399066, u_supplier_id#10399067, u_declaring_supplier_id#10399068, u_supplier_type#10399069, u_threshold#10399070, u_normalized_name#10399071, e_supplier_id#10399089, e_canonical_id#10399090, e_normalized_name#10399091, e_create_date#10399093]
Input [11]: [u_brand_context_id#10399066, u_supplier_id#10399067, u_declaring_supplier_id#10399068, u_supplier_type#10399069, u_threshold#10399070, u_normalized_name#10399071, e_supplier_id#10399089, e_canonical_id#10399090, e_normalized_name#10399091, e_create_date#10399093, pythonUDF0#10400694]
(63) BatchEvalPython
Input [10]: [u_brand_context_id#10399066, u_supplier_id#10399067, u_declaring_supplier_id#10399068, u_supplier_type#10399069, u_threshold#10399070, u_normalized_name#10399071, e_supplier_id#10399089, e_canonical_id#10399090, e_normalized_name#10399091, e_create_date#10399093]
Arguments: [calculate_similarity(u_normalized_name#10399071, e_normalized_name#10399091)#10399146], [pythonUDF0#10400695]
(64) Project
Output [9]: [u_brand_context_id#10399066, u_supplier_id#10399067, u_declaring_supplier_id#10399068, u_supplier_type#10399069, u_threshold#10399070, e_supplier_id#10399089, e_canonical_id#10399090, e_create_date#10399093, pythonUDF0#10400695 AS similarity#10399147]
Input [11]: [u_brand_context_id#10399066, u_supplier_id#10399067, u_declaring_supplier_id#10399068, u_supplier_type#10399069, u_threshold#10399070, u_normalized_name#10399071, e_supplier_id#10399089, e_canonical_id#10399090, e_normalized_name#10399091, e_create_date#10399093, pythonUDF0#10400695]
(65) Sort
Input [9]: [u_brand_context_id#10399066, u_supplier_id#10399067, u_declaring_supplier_id#10399068, u_supplier_type#10399069, u_threshold#10399070, e_supplier_id#10399089, e_canonical_id#10399090, e_create_date#10399093, similarity#10399147]
Arguments: [u_brand_context_id#10399066 ASC NULLS FIRST, u_supplier_id#10399067 ASC NULLS FIRST, similarity#10399147 DESC NULLS LAST, e_create_date#10399093 ASC NULLS FIRST, e_supplier_id#10399089 ASC NULLS FIRST], false, 0
(66) WindowGroupLimit
Input [9]: [u_brand_context_id#10399066, u_supplier_id#10399067, u_declaring_supplier_id#10399068, u_supplier_type#10399069, u_threshold#10399070, e_supplier_id#10399089, e_canonical_id#10399090, e_create_date#10399093, similarity#10399147]
Arguments: [u_brand_context_id#10399066, u_supplier_id#10399067], [similarity#10399147 DESC NULLS LAST, e_create_date#10399093 ASC NULLS FIRST, e_supplier_id#10399089 ASC NULLS FIRST], row_number(), 1, Partial
(67) Exchange
Input [9]: [u_brand_context_id#10399066, u_supplier_id#10399067, u_declaring_supplier_id#10399068, u_supplier_type#10399069, u_threshold#10399070, e_supplier_id#10399089, e_canonical_id#10399090, e_create_date#10399093, similarity#10399147]
Arguments: hashpartitioning(u_brand_context_id#10399066, u_supplier_id#10399067, 10), ENSURE_REQUIREMENTS, [plan_id=361568]
(68) Sort
Input [9]: [u_brand_context_id#10399066, u_supplier_id#10399067, u_declaring_supplier_id#10399068, u_supplier_type#10399069, u_threshold#10399070, e_supplier_id#10399089, e_canonical_id#10399090, e_create_date#10399093, similarity#10399147]
Arguments: [u_brand_context_id#10399066 ASC NULLS FIRST, u_supplier_id#10399067 ASC NULLS FIRST, similarity#10399147 DESC NULLS LAST, e_create_date#10399093 ASC NULLS FIRST, e_supplier_id#10399089 ASC NULLS FIRST], false, 0
(69) WindowGroupLimit
Input [9]: [u_brand_context_id#10399066, u_supplier_id#10399067, u_declaring_supplier_id#10399068, u_supplier_type#10399069, u_threshold#10399070, e_supplier_id#10399089, e_canonical_id#10399090, e_create_date#10399093, similarity#10399147]
Arguments: [u_brand_context_id#10399066, u_supplier_id#10399067], [similarity#10399147 DESC NULLS LAST, e_create_date#10399093 ASC NULLS FIRST, e_supplier_id#10399089 ASC NULLS FIRST], row_number(), 1, Final
(70) Window
Input [9]: [u_brand_context_id#10399066, u_supplier_id#10399067, u_declaring_supplier_id#10399068, u_supplier_type#10399069, u_threshold#10399070, e_supplier_id#10399089, e_canonical_id#10399090, e_create_date#10399093, similarity#10399147]
Arguments: [row_number() windowspecdefinition(u_brand_context_id#10399066, u_supplier_id#10399067, similarity#10399147 DESC NULLS LAST, e_create_date#10399093 ASC NULLS FIRST, e_supplier_id#10399089 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank#10399167], [u_brand_context_id#10399066, u_supplier_id#10399067], [similarity#10399147 DESC NULLS LAST, e_create_date#10399093 ASC NULLS FIRST, e_supplier_id#10399089 ASC NULLS FIRST]
(71) Filter
Input [10]: [u_brand_context_id#10399066, u_supplier_id#10399067, u_declaring_supplier_id#10399068, u_supplier_type#10399069, u_threshold#10399070, e_supplier_id#10399089, e_canonical_id#10399090, e_create_date#10399093, similarity#10399147, rank#10399167]
Condition : (rank#10399167 = 1)
(72) Project
Output [10]: [u_brand_context_id#10399066 AS brand_context_id#10399187, u_supplier_id#10399067 AS supplier_id#10399188, u_declaring_supplier_id#10399068 AS declaring_supplier_id#10399189, u_supplier_type#10399069 AS supplier_type#10399190, e_canonical_id#10399090 AS canonical_id#10399191, 0 AS is_primary#10399192, u_threshold#10399070 AS configured_threshold#10399193, cast((similarity#10399147 * 100.0) as double) AS match_percentage#10399300, cast(similarity#10399147 as double) AS confidence_score#10399301, matched_existing AS match_reason#10399196]
Input [10]: [u_brand_context_id#10399066, u_supplier_id#10399067, u_declaring_supplier_id#10399068, u_supplier_type#10399069, u_threshold#10399070, e_supplier_id#10399089, e_canonical_id#10399090, e_create_date#10399093, similarity#10399147, rank#10399167]
(73) InMemoryTableScan
Output [5]: [brand_context_id#10399312, supplier_id#10399313, supplier_type#10399318, declaring_supplier_id#10399319, threshold#10399322]
Arguments: [brand_context_id#10399312, supplier_id#10399313, supplier_type#10399318, declaring_supplier_id#10399319, threshold#10399322]
(74) InMemoryRelation
Arguments: [brand_context_id#10399312, supplier_id#10399313, entity_name#10399314, city#10399315, state#10399316, country#10399317, supplier_type#10399318, declaring_supplier_id#10399319, threshold#10399322, create_date#10399323], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@7795a5ad,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Project [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, threshold#8983070, create_date#8983071]
+- *(1) Filter isnull(canonical_id#8983068)
+- TableCacheQueryStage 0
+- InMemoryTableScan [brand_context_id#8983060, canonical_id#8983068, city#8983063, country#8983065, create_date#8983071, declaring_supplier_id#8983067, entity_name#8983062, state#8983064, supplier_id#8983061, supplier_type#8983066, threshold#8983070], [isnull(canonical_id#8983068)]
+- InMemoryRelation [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, canonical_id#8983068, is_primary#8983069, threshold#8983070, create_date#8983071], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#8983060,supplier_id#8983061,entity_name#8983062,city#8983063,state#8983064,country#8983065,supplier_type#8983066,declaring_supplier_id#8983067,canonical_id#8983068,is_primary#8983069,threshold#8983070,create_date#8983071] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Project [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, threshold#8983070, create_date#8983071]
+- Filter isnull(canonical_id#8983068)
+- InMemoryTableScan [brand_context_id#8983060, canonical_id#8983068, city#8983063, country#8983065, create_date#8983071, declaring_supplier_id#8983067, entity_name#8983062, state#8983064, supplier_id#8983061, supplier_type#8983066, threshold#8983070], [isnull(canonical_id#8983068)]
+- InMemoryRelation [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, canonical_id#8983068, is_primary#8983069, threshold#8983070, create_date#8983071], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#8983060,supplier_id#8983061,entity_name#8983062,city#8983063,state#8983064,country#8983065,supplier_type#8983066,declaring_supplier_id#8983067,canonical_id#8983068,is_primary#8983069,threshold#8983070,create_date#8983071] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(75) AdaptiveSparkPlan
Output [10]: [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, threshold#8983070, create_date#8983071]
Arguments: isFinalPlan=false
(76) Exchange
Input [5]: [brand_context_id#10399312, supplier_id#10399313, supplier_type#10399318, declaring_supplier_id#10399319, threshold#10399322]
Arguments: hashpartitioning(brand_context_id#10399312, supplier_id#10399313, 10), ENSURE_REQUIREMENTS, [plan_id=361579]
(77) Sort
Input [5]: [brand_context_id#10399312, supplier_id#10399313, supplier_type#10399318, declaring_supplier_id#10399319, threshold#10399322]
Arguments: [brand_context_id#10399312 ASC NULLS FIRST, supplier_id#10399313 ASC NULLS FIRST], false, 0
(78) InMemoryTableScan
Output [2]: [brand_context_id#10399324, supplier_id#10399325]
Arguments: [brand_context_id#10399324, supplier_id#10399325], [isnotnull(brand_context_id#10399324), isnotnull(supplier_id#10399325)]
(79) InMemoryRelation
Arguments: [brand_context_id#10399324, supplier_id#10399325, entity_name#10399326, city#10399327, state#10399328, country#10399329, supplier_type#10399330, declaring_supplier_id#10399331, threshold#10399334, create_date#10399335], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@7795a5ad,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Project [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, threshold#8983070, create_date#8983071]
+- *(1) Filter isnull(canonical_id#8983068)
+- TableCacheQueryStage 0
+- InMemoryTableScan [brand_context_id#8983060, canonical_id#8983068, city#8983063, country#8983065, create_date#8983071, declaring_supplier_id#8983067, entity_name#8983062, state#8983064, supplier_id#8983061, supplier_type#8983066, threshold#8983070], [isnull(canonical_id#8983068)]
+- InMemoryRelation [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, canonical_id#8983068, is_primary#8983069, threshold#8983070, create_date#8983071], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#8983060,supplier_id#8983061,entity_name#8983062,city#8983063,state#8983064,country#8983065,supplier_type#8983066,declaring_supplier_id#8983067,canonical_id#8983068,is_primary#8983069,threshold#8983070,create_date#8983071] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Project [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, threshold#8983070, create_date#8983071]
+- Filter isnull(canonical_id#8983068)
+- InMemoryTableScan [brand_context_id#8983060, canonical_id#8983068, city#8983063, country#8983065, create_date#8983071, declaring_supplier_id#8983067, entity_name#8983062, state#8983064, supplier_id#8983061, supplier_type#8983066, threshold#8983070], [isnull(canonical_id#8983068)]
+- InMemoryRelation [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, canonical_id#8983068, is_primary#8983069, threshold#8983070, create_date#8983071], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#8983060,supplier_id#8983061,entity_name#8983062,city#8983063,state#8983064,country#8983065,supplier_type#8983066,declaring_supplier_id#8983067,canonical_id#8983068,is_primary#8983069,threshold#8983070,create_date#8983071] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(80) AdaptiveSparkPlan
Output [10]: [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, threshold#8983070, create_date#8983071]
Arguments: isFinalPlan=false
(81) Filter
Input [2]: [brand_context_id#10399324, supplier_id#10399325]
Condition : (isnotnull(brand_context_id#10399324) AND isnotnull(supplier_id#10399325))
(82) BroadcastExchange
Input [2]: [brand_context_id#10399324, supplier_id#10399325]
Arguments: HashedRelationBroadcastMode(List(input[0, string, false], input[1, string, false]),false), [plan_id=361574]
(83) InMemoryTableScan
Output [4]: [brand_context_id#10399336, canonical_id#10399344, is_primary#10399345, supplier_id#10399337]
Arguments: [brand_context_id#10399336, canonical_id#10399344, is_primary#10399345, supplier_id#10399337], [isnotnull(is_primary#10399345), is_primary#10399345, isnotnull(canonical_id#10399344), isnotnull(brand_context_id#10399336), isnotnull(supplier_id#10399337)]
(84) InMemoryRelation
Arguments: [brand_context_id#10399336, supplier_id#10399337, canonical_id#10399344, is_primary#10399345, entity_name#10399338, city#10399339, state#10399340, country#10399341, threshold#10399346, create_date#10399347], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@7795a5ad,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Filter isnotnull(canonical_id#8983068)
+- TableCacheQueryStage 0
+- InMemoryTableScan [brand_context_id#8983060, supplier_id#8983061, canonical_id#8983068, is_primary#8983069, entity_name#8983062, city#8983063, state#8983064, country#8983065, threshold#8983070, create_date#8983071], [isnotnull(canonical_id#8983068)]
+- InMemoryRelation [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, canonical_id#8983068, is_primary#8983069, threshold#8983070, create_date#8983071], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#8983060,supplier_id#8983061,entity_name#8983062,city#8983063,state#8983064,country#8983065,supplier_type#8983066,declaring_supplier_id#8983067,canonical_id#8983068,is_primary#8983069,threshold#8983070,create_date#8983071] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Filter isnotnull(canonical_id#8983068)
+- InMemoryTableScan [brand_context_id#8983060, supplier_id#8983061, canonical_id#8983068, is_primary#8983069, entity_name#8983062, city#8983063, state#8983064, country#8983065, threshold#8983070, create_date#8983071], [isnotnull(canonical_id#8983068)]
+- InMemoryRelation [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, canonical_id#8983068, is_primary#8983069, threshold#8983070, create_date#8983071], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#8983060,supplier_id#8983061,entity_name#8983062,city#8983063,state#8983064,country#8983065,supplier_type#8983066,declaring_supplier_id#8983067,canonical_id#8983068,is_primary#8983069,threshold#8983070,create_date#8983071] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(85) AdaptiveSparkPlan
Output [10]: [brand_context_id#8983060, supplier_id#8983061, canonical_id#8983068, is_primary#8983069, entity_name#8983062, city#8983063, state#8983064, country#8983065, threshold#8983070, create_date#8983071]
Arguments: isFinalPlan=false
(86) Filter
Input [4]: [brand_context_id#10399336, canonical_id#10399344, is_primary#10399345, supplier_id#10399337]
Condition : ((((isnotnull(is_primary#10399345) AND is_primary#10399345) AND isnotnull(canonical_id#10399344)) AND isnotnull(brand_context_id#10399336)) AND isnotnull(supplier_id#10399337))
(87) Project
Output [2]: [brand_context_id#10399336, supplier_id#10399337]
Input [4]: [brand_context_id#10399336, canonical_id#10399344, is_primary#10399345, supplier_id#10399337]
(88) BroadcastHashJoin
Left keys [2]: [brand_context_id#10399324, supplier_id#10399325]
Right keys [2]: [brand_context_id#10399336, supplier_id#10399337]
Join type: Inner
Join condition: None
(89) Project
Output [2]: [brand_context_id#10399324, supplier_id#10399325]
Input [4]: [brand_context_id#10399324, supplier_id#10399325, brand_context_id#10399336, supplier_id#10399337]
(90) Exchange
Input [2]: [brand_context_id#10399324, supplier_id#10399325]
Arguments: hashpartitioning(brand_context_id#10399324, supplier_id#10399325, 10), ENSURE_REQUIREMENTS, [plan_id=361580]
(91) Sort
Input [2]: [brand_context_id#10399324, supplier_id#10399325]
Arguments: [brand_context_id#10399324 ASC NULLS FIRST, supplier_id#10399325 ASC NULLS FIRST], false, 0
(92) SortMergeJoin
Left keys [2]: [brand_context_id#10399312, supplier_id#10399313]
Right keys [2]: [brand_context_id#10399324, supplier_id#10399325]
Join type: LeftAnti
Join condition: None
(93) InMemoryTableScan
Output [7]: [brand_context_id#10399209, supplier_id#10399210, entity_name#10399211, city#10399212, state#10399213, country#10399214, threshold#10399219]
Arguments: [brand_context_id#10399209, supplier_id#10399210, entity_name#10399211, city#10399212, state#10399213, country#10399214, threshold#10399219], [isnotnull(threshold#10399219), isnotnull(brand_context_id#10399209), isnotnull(supplier_id#10399210)]
(94) InMemoryRelation
Arguments: [brand_context_id#10399209, supplier_id#10399210, entity_name#10399211, city#10399212, state#10399213, country#10399214, supplier_type#10399215, declaring_supplier_id#10399216, threshold#10399219, create_date#10399220], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@7795a5ad,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Project [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, threshold#8983070, create_date#8983071]
+- *(1) Filter isnull(canonical_id#8983068)
+- TableCacheQueryStage 0
+- InMemoryTableScan [brand_context_id#8983060, canonical_id#8983068, city#8983063, country#8983065, create_date#8983071, declaring_supplier_id#8983067, entity_name#8983062, state#8983064, supplier_id#8983061, supplier_type#8983066, threshold#8983070], [isnull(canonical_id#8983068)]
+- InMemoryRelation [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, canonical_id#8983068, is_primary#8983069, threshold#8983070, create_date#8983071], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#8983060,supplier_id#8983061,entity_name#8983062,city#8983063,state#8983064,country#8983065,supplier_type#8983066,declaring_supplier_id#8983067,canonical_id#8983068,is_primary#8983069,threshold#8983070,create_date#8983071] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Project [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, threshold#8983070, create_date#8983071]
+- Filter isnull(canonical_id#8983068)
+- InMemoryTableScan [brand_context_id#8983060, canonical_id#8983068, city#8983063, country#8983065, create_date#8983071, declaring_supplier_id#8983067, entity_name#8983062, state#8983064, supplier_id#8983061, supplier_type#8983066, threshold#8983070], [isnull(canonical_id#8983068)]
+- InMemoryRelation [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, canonical_id#8983068, is_primary#8983069, threshold#8983070, create_date#8983071], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#8983060,supplier_id#8983061,entity_name#8983062,city#8983063,state#8983064,country#8983065,supplier_type#8983066,declaring_supplier_id#8983067,canonical_id#8983068,is_primary#8983069,threshold#8983070,create_date#8983071] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(95) AdaptiveSparkPlan
Output [10]: [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, threshold#8983070, create_date#8983071]
Arguments: isFinalPlan=false
(96) Filter
Input [7]: [brand_context_id#10399209, supplier_id#10399210, entity_name#10399211, city#10399212, state#10399213, country#10399214, threshold#10399219]
Condition : ((isnotnull(threshold#10399219) AND isnotnull(brand_context_id#10399209)) AND isnotnull(supplier_id#10399210))
(97) BatchEvalPython
Input [7]: [brand_context_id#10399209, supplier_id#10399210, entity_name#10399211, city#10399212, state#10399213, country#10399214, threshold#10399219]
Arguments: [normalize_location(city#10399212)#10395260, normalize_location(state#10399213)#10395274, normalize_location(country#10399214)#10395289], [pythonUDF0#10400696, pythonUDF1#10400697, pythonUDF2#10400698]
(98) Filter
Input [10]: [brand_context_id#10399209, supplier_id#10399210, entity_name#10399211, city#10399212, state#10399213, country#10399214, threshold#10399219, pythonUDF0#10400696, pythonUDF1#10400697, pythonUDF2#10400698]
Condition : ((isnotnull(pythonUDF0#10400696) AND isnotnull(pythonUDF1#10400697)) AND isnotnull(pythonUDF2#10400698))
(99) Project
Output [7]: [brand_context_id#10399209, supplier_id#10399210, entity_name#10399211, city#10399212, state#10399213, country#10399214, threshold#10399219]
Input [10]: [brand_context_id#10399209, supplier_id#10399210, entity_name#10399211, city#10399212, state#10399213, country#10399214, threshold#10399219, pythonUDF0#10400696, pythonUDF1#10400697, pythonUDF2#10400698]
(100) BatchEvalPython
Input [7]: [brand_context_id#10399209, supplier_id#10399210, entity_name#10399211, city#10399212, state#10399213, country#10399214, threshold#10399219]
Arguments: [normalize_name(entity_name#10399211)#10395247, normalize_location(city#10399212)#10395260, normalize_location(state#10399213)#10395274, normalize_location(country#10399214)#10395289], [pythonUDF0#10400699, pythonUDF1#10400700, pythonUDF2#10400701, pythonUDF3#10400702]
(101) Project
Output [5]: [brand_context_id#10399209, supplier_id#10399210, threshold#10399219, pythonUDF0#10400699 AS normalized_name#10395248, concat_ws(|, coalesce(pythonUDF1#10400700, ), coalesce(pythonUDF2#10400701, ), coalesce(pythonUDF3#10400702, )) AS location_key#10395305]
Input [11]: [brand_context_id#10399209, supplier_id#10399210, entity_name#10399211, city#10399212, state#10399213, country#10399214, threshold#10399219, pythonUDF0#10400699, pythonUDF1#10400700, pythonUDF2#10400701, pythonUDF3#10400702]
(102) Exchange
Input [5]: [brand_context_id#10399209, supplier_id#10399210, threshold#10399219, normalized_name#10395248, location_key#10395305]
Arguments: hashpartitioning(brand_context_id#10399209, supplier_id#10399210, 10), ENSURE_REQUIREMENTS, [plan_id=361589]
(103) Sort
Input [5]: [brand_context_id#10399209, supplier_id#10399210, threshold#10399219, normalized_name#10395248, location_key#10395305]
Arguments: [brand_context_id#10399209 ASC NULLS FIRST, supplier_id#10399210 ASC NULLS FIRST], false, 0
(104) InMemoryTableScan
Output [2]: [brand_context_id#10399221, supplier_id#10399222]
Arguments: [brand_context_id#10399221, supplier_id#10399222], [isnotnull(brand_context_id#10399221), isnotnull(supplier_id#10399222)]
(105) InMemoryRelation
Arguments: [brand_context_id#10399221, supplier_id#10399222, entity_name#10399223, city#10399224, state#10399225, country#10399226, supplier_type#10399227, declaring_supplier_id#10399228, threshold#10399231, create_date#10399232], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@7795a5ad,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Project [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, threshold#8983070, create_date#8983071]
+- *(1) Filter isnull(canonical_id#8983068)
+- TableCacheQueryStage 0
+- InMemoryTableScan [brand_context_id#8983060, canonical_id#8983068, city#8983063, country#8983065, create_date#8983071, declaring_supplier_id#8983067, entity_name#8983062, state#8983064, supplier_id#8983061, supplier_type#8983066, threshold#8983070], [isnull(canonical_id#8983068)]
+- InMemoryRelation [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, canonical_id#8983068, is_primary#8983069, threshold#8983070, create_date#8983071], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#8983060,supplier_id#8983061,entity_name#8983062,city#8983063,state#8983064,country#8983065,supplier_type#8983066,declaring_supplier_id#8983067,canonical_id#8983068,is_primary#8983069,threshold#8983070,create_date#8983071] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Project [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, threshold#8983070, create_date#8983071]
+- Filter isnull(canonical_id#8983068)
+- InMemoryTableScan [brand_context_id#8983060, canonical_id#8983068, city#8983063, country#8983065, create_date#8983071, declaring_supplier_id#8983067, entity_name#8983062, state#8983064, supplier_id#8983061, supplier_type#8983066, threshold#8983070], [isnull(canonical_id#8983068)]
+- InMemoryRelation [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, canonical_id#8983068, is_primary#8983069, threshold#8983070, create_date#8983071], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#8983060,supplier_id#8983061,entity_name#8983062,city#8983063,state#8983064,country#8983065,supplier_type#8983066,declaring_supplier_id#8983067,canonical_id#8983068,is_primary#8983069,threshold#8983070,create_date#8983071] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(106) AdaptiveSparkPlan
Output [10]: [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, threshold#8983070, create_date#8983071]
Arguments: isFinalPlan=false
(107) Filter
Input [2]: [brand_context_id#10399221, supplier_id#10399222]
Condition : (isnotnull(brand_context_id#10399221) AND isnotnull(supplier_id#10399222))
(108) BroadcastExchange
Input [2]: [brand_context_id#10399221, supplier_id#10399222]
Arguments: HashedRelationBroadcastMode(List(input[0, string, false], input[1, string, false]),false), [plan_id=361584]
(109) InMemoryTableScan
Output [4]: [brand_context_id#10399233, canonical_id#10399241, is_primary#10399242, supplier_id#10399234]
Arguments: [brand_context_id#10399233, canonical_id#10399241, is_primary#10399242, supplier_id#10399234], [isnotnull(is_primary#10399242), is_primary#10399242, isnotnull(canonical_id#10399241), isnotnull(brand_context_id#10399233), isnotnull(supplier_id#10399234)]
(110) InMemoryRelation
Arguments: [brand_context_id#10399233, supplier_id#10399234, canonical_id#10399241, is_primary#10399242, entity_name#10399235, city#10399236, state#10399237, country#10399238, threshold#10399243, create_date#10399244], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@7795a5ad,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Filter isnotnull(canonical_id#8983068)
+- TableCacheQueryStage 0
+- InMemoryTableScan [brand_context_id#8983060, supplier_id#8983061, canonical_id#8983068, is_primary#8983069, entity_name#8983062, city#8983063, state#8983064, country#8983065, threshold#8983070, create_date#8983071], [isnotnull(canonical_id#8983068)]
+- InMemoryRelation [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, canonical_id#8983068, is_primary#8983069, threshold#8983070, create_date#8983071], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#8983060,supplier_id#8983061,entity_name#8983062,city#8983063,state#8983064,country#8983065,supplier_type#8983066,declaring_supplier_id#8983067,canonical_id#8983068,is_primary#8983069,threshold#8983070,create_date#8983071] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Filter isnotnull(canonical_id#8983068)
+- InMemoryTableScan [brand_context_id#8983060, supplier_id#8983061, canonical_id#8983068, is_primary#8983069, entity_name#8983062, city#8983063, state#8983064, country#8983065, threshold#8983070, create_date#8983071], [isnotnull(canonical_id#8983068)]
+- InMemoryRelation [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, canonical_id#8983068, is_primary#8983069, threshold#8983070, create_date#8983071], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#8983060,supplier_id#8983061,entity_name#8983062,city#8983063,state#8983064,country#8983065,supplier_type#8983066,declaring_supplier_id#8983067,canonical_id#8983068,is_primary#8983069,threshold#8983070,create_date#8983071] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(111) AdaptiveSparkPlan
Output [10]: [brand_context_id#8983060, supplier_id#8983061, canonical_id#8983068, is_primary#8983069, entity_name#8983062, city#8983063, state#8983064, country#8983065, threshold#8983070, create_date#8983071]
Arguments: isFinalPlan=false
(112) Filter
Input [4]: [brand_context_id#10399233, canonical_id#10399241, is_primary#10399242, supplier_id#10399234]
Condition : ((((isnotnull(is_primary#10399242) AND is_primary#10399242) AND isnotnull(canonical_id#10399241)) AND isnotnull(brand_context_id#10399233)) AND isnotnull(supplier_id#10399234))
(113) Project
Output [2]: [brand_context_id#10399233, supplier_id#10399234]
Input [4]: [brand_context_id#10399233, canonical_id#10399241, is_primary#10399242, supplier_id#10399234]
(114) BroadcastHashJoin
Left keys [2]: [brand_context_id#10399221, supplier_id#10399222]
Right keys [2]: [brand_context_id#10399233, supplier_id#10399234]
Join type: Inner
Join condition: None
(115) Project
Output [2]: [brand_context_id#10399221, supplier_id#10399222]
Input [4]: [brand_context_id#10399221, supplier_id#10399222, brand_context_id#10399233, supplier_id#10399234]
(116) Exchange
Input [2]: [brand_context_id#10399221, supplier_id#10399222]
Arguments: hashpartitioning(brand_context_id#10399221, supplier_id#10399222, 10), ENSURE_REQUIREMENTS, [plan_id=361590]
(117) Sort
Input [2]: [brand_context_id#10399221, supplier_id#10399222]
Arguments: [brand_context_id#10399221 ASC NULLS FIRST, supplier_id#10399222 ASC NULLS FIRST], false, 0
(118) SortMergeJoin
Left keys [2]: [brand_context_id#10399209, supplier_id#10399210]
Right keys [2]: [brand_context_id#10399221, supplier_id#10399222]
Join type: LeftAnti
Join condition: None
(119) Project
Output [5]: [brand_context_id#10399209 AS u_brand_context_id#10399066, supplier_id#10399210 AS u_supplier_id#10399067, threshold#10399219 AS u_threshold#10399070, normalized_name#10395248 AS u_normalized_name#10399071, location_key#10395305 AS u_location_key#10399075]
Input [5]: [brand_context_id#10399209, supplier_id#10399210, threshold#10399219, normalized_name#10395248, location_key#10395305]
(120) BroadcastExchange
Input [5]: [u_brand_context_id#10399066, u_supplier_id#10399067, u_threshold#10399070, u_normalized_name#10399071, u_location_key#10399075]
Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[4, string, false]),false), [plan_id=361596]
(121) InMemoryTableScan
Output [7]: [brand_context_id#10399348, supplier_id#10399349, entity_name#10399350, city#10399351, state#10399352, country#10399353, create_date#10399359]
Arguments: [brand_context_id#10399348, supplier_id#10399349, entity_name#10399350, city#10399351, state#10399352, country#10399353, create_date#10399359], [isnotnull(brand_context_id#10399348)]
(122) InMemoryRelation
Arguments: [brand_context_id#10399348, supplier_id#10399349, canonical_id#10399356, is_primary#10399357, entity_name#10399350, city#10399351, state#10399352, country#10399353, threshold#10399358, create_date#10399359], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@7795a5ad,StorageLevel(disk, memory, deserialized, 1 replicas),AdaptiveSparkPlan isFinalPlan=true
+- == Final Plan ==
*(1) Filter isnotnull(canonical_id#8983068)
+- TableCacheQueryStage 0
+- InMemoryTableScan [brand_context_id#8983060, supplier_id#8983061, canonical_id#8983068, is_primary#8983069, entity_name#8983062, city#8983063, state#8983064, country#8983065, threshold#8983070, create_date#8983071], [isnotnull(canonical_id#8983068)]
+- InMemoryRelation [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, canonical_id#8983068, is_primary#8983069, threshold#8983070, create_date#8983071], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#8983060,supplier_id#8983061,entity_name#8983062,city#8983063,state#8983064,country#8983065,supplier_type#8983066,declaring_supplier_id#8983067,canonical_id#8983068,is_primary#8983069,threshold#8983070,create_date#8983071] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
+- == Initial Plan ==
Filter isnotnull(canonical_id#8983068)
+- InMemoryTableScan [brand_context_id#8983060, supplier_id#8983061, canonical_id#8983068, is_primary#8983069, entity_name#8983062, city#8983063, state#8983064, country#8983065, threshold#8983070, create_date#8983071], [isnotnull(canonical_id#8983068)]
+- InMemoryRelation [brand_context_id#8983060, supplier_id#8983061, entity_name#8983062, city#8983063, state#8983064, country#8983065, supplier_type#8983066, declaring_supplier_id#8983067, canonical_id#8983068, is_primary#8983069, threshold#8983070, create_date#8983071], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Scan JDBCRelation((
SELECT DISTINCT
v.brand_context_id,
v.supplier_id,
v.entity_name,
v.city,
v.state,
v.country,
v.supplier_type,
v.declaring_supplier_id,
v.canonical_id,
v.is_primary,
COALESCE(v.threshold, 0.9) AS threshold,
v.create_date
FROM vw_all_suppliers_for_dedup v
) AS all_suppliers) [numPartitions=1] [brand_context_id#8983060,supplier_id#8983061,entity_name#8983062,city#8983063,state#8983064,country#8983065,supplier_type#8983066,declaring_supplier_id#8983067,canonical_id#8983068,is_primary#8983069,threshold#8983070,create_date#8983071] PushedFilters: [], ReadSchema: struct<brand_context_id:string,supplier_id:string,entity_name:string,city:string,state:string,cou...
,None)
(123) AdaptiveSparkPlan
Output [10]: [brand_context_id#8983060, supplier_id#8983061, canonical_id#8983068, is_primary#8983069, entity_name#8983062, city#8983063, state#8983064, country#8983065, threshold#8983070, create_date#8983071]
Arguments: isFinalPlan=false
(124) Filter
Input [7]: [brand_context_id#10399348, supplier_id#10399349, entity_name#10399350, city#10399351, state#10399352, country#10399353, create_date#10399359]
Condition : isnotnull(brand_context_id#10399348)
(125) BatchEvalPython
Input [7]: [brand_context_id#10399348, supplier_id#10399349, entity_name#10399350, city#10399351, state#10399352, country#10399353, create_date#10399359]
Arguments: [normalize_name(entity_name#10399350)#10395321, normalize_location(city#10399351)#10395334, normalize_location(state#10399352)#10395348, normalize_location(country#10399353)#10395363], [pythonUDF0#10400703, pythonUDF1#10400704, pythonUDF2#10400705, pythonUDF3#10400706]
(126) Project
Output [5]: [brand_context_id#10399348 AS e_brand_context_id#10399088, supplier_id#10399349 AS e_supplier_id#10399089, pythonUDF0#10400703 AS e_normalized_name#10399091, concat_ws(|, coalesce(pythonUDF1#10400704, ), coalesce(pythonUDF2#10400705, ), coalesce(pythonUDF3#10400706, )) AS e_location_key#10399092, create_date#10399359 AS e_create_date#10399093]
Input [11]: [brand_context_id#10399348, supplier_id#10399349, entity_name#10399350, city#10399351, state#10399352, country#10399353, create_date#10399359, pythonUDF0#10400703, pythonUDF1#10400704, pythonUDF2#10400705, pythonUDF3#10400706]
(127) BroadcastHashJoin
Left keys [2]: [u_brand_context_id#10399066, u_location_key#10399075]
Right keys [2]: [e_brand_context_id#10399088, e_location_key#10399092]
Join type: Inner
Join condition: None
(128) Project
Output [7]: [u_brand_context_id#10399066, u_supplier_id#10399067, u_threshold#10399070, u_normalized_name#10399071, e_supplier_id#10399089, e_normalized_name#10399091, e_create_date#10399093]
Input [10]: [u_brand_context_id#10399066, u_supplier_id#10399067, u_threshold#10399070, u_normalized_name#10399071, u_location_key#10399075, e_brand_context_id#10399088, e_supplier_id#10399089, e_normalized_name#10399091, e_location_key#10399092, e_create_date#10399093]
(129) BatchEvalPython
Input [7]: [u_brand_context_id#10399066, u_supplier_id#10399067, u_threshold#10399070, u_normalized_name#10399071, e_supplier_id#10399089, e_normalized_name#10399091, e_create_date#10399093]
Arguments: [calculate_similarity(u_normalized_name#10399071, e_normalized_name#10399091)#10399146], [pythonUDF0#10400707]
(130) Filter
Input [8]: [u_brand_context_id#10399066, u_supplier_id#10399067, u_threshold#10399070, u_normalized_name#10399071, e_supplier_id#10399089, e_normalized_name#10399091, e_create_date#10399093, pythonUDF0#10400707]
Condition : (cast(pythonUDF0#10400707 as double) >= u_threshold#10399070)
(131) Project
Output [6]: [u_brand_context_id#10399066, u_supplier_id#10399067, u_normalized_name#10399071, e_supplier_id#10399089, e_normalized_name#10399091, e_create_date#10399093]
Input [8]: [u_brand_context_id#10399066, u_supplier_id#10399067, u_threshold#10399070, u_normalized_name#10399071, e_supplier_id#10399089, e_normalized_name#10399091, e_create_date#10399093, pythonUDF0#10400707]
(132) BatchEvalPython
Input [6]: [u_brand_context_id#10399066, u_supplier_id#10399067, u_normalized_name#10399071, e_supplier_id#10399089, e_normalized_name#10399091, e_create_date#10399093]
Arguments: [calculate_similarity(u_normalized_name#10399071, e_normalized_name#10399091)#10399146], [pythonUDF0#10400708]
(133) Project
Output [5]: [u_brand_context_id#10399066, u_supplier_id#10399067, e_supplier_id#10399089, e_create_date#10399093, pythonUDF0#10400708 AS similarity#10399147]
Input [7]: [u_brand_context_id#10399066, u_supplier_id#10399067, u_normalized_name#10399071, e_supplier_id#10399089, e_normalized_name#10399091, e_create_date#10399093, pythonUDF0#10400708]
(134) Sort
Input [5]: [u_brand_context_id#10399066, u_supplier_id#10399067, e_supplier_id#10399089, e_create_date#10399093, similarity#10399147]
Arguments: [u_brand_context_id#10399066 ASC NULLS FIRST, u_supplier_id#10399067 ASC NULLS FIRST, similarity#10399147 DESC NULLS LAST, e_create_date#10399093 ASC NULLS FIRST, e_supplier_id#10399089 ASC NULLS FIRST], false, 0
(135) WindowGroupLimit
Input [5]: [u_brand_context_id#10399066, u_supplier_id#10399067, e_supplier_id#10399089, e_create_date#10399093, similarity#10399147]
Arguments: [u_brand_context_id#10399066, u_supplier_id#10399067], [similarity#10399147 DESC NULLS LAST, e_create_date#10399093 ASC NULLS FIRST, e_supplier_id#10399089 ASC NULLS FIRST], row_number(), 1, Partial
(136) Exchange
Input [5]: [u_brand_context_id#10399066, u_supplier_id#10399067, e_supplier_id#10399089, e_create_date#10399093, similarity#10399147]
Arguments: hashpartitioning(u_brand_context_id#10399066, u_supplier_id#10399067, 10), ENSURE_REQUIREMENTS, [plan_id=361608]
(137) Sort
Input [5]: [u_brand_context_id#10399066, u_supplier_id#10399067, e_supplier_id#10399089, e_create_date#10399093, similarity#10399147]
Arguments: [u_brand_context_id#10399066 ASC NULLS FIRST, u_supplier_id#10399067 ASC NULLS FIRST, similarity#10399147 DESC NULLS LAST, e_create_date#10399093 ASC NULLS FIRST, e_supplier_id#10399089 ASC NULLS FIRST], false, 0
(138) WindowGroupLimit
Input [5]: [u_brand_context_id#10399066, u_supplier_id#10399067, e_supplier_id#10399089, e_create_date#10399093, similarity#10399147]
Arguments: [u_brand_context_id#10399066, u_supplier_id#10399067], [similarity#10399147 DESC NULLS LAST, e_create_date#10399093 ASC NULLS FIRST, e_supplier_id#10399089 ASC NULLS FIRST], row_number(), 1, Final
(139) Window
Input [5]: [u_brand_context_id#10399066, u_supplier_id#10399067, e_supplier_id#10399089, e_create_date#10399093, similarity#10399147]
Arguments: [row_number() windowspecdefinition(u_brand_context_id#10399066, u_supplier_id#10399067, similarity#10399147 DESC NULLS LAST, e_create_date#10399093 ASC NULLS FIRST, e_supplier_id#10399089 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank#10399167], [u_brand_context_id#10399066, u_supplier_id#10399067], [similarity#10399147 DESC NULLS LAST, e_create_date#10399093 ASC NULLS FIRST, e_supplier_id#10399089 ASC NULLS FIRST]
(140) Filter
Input [6]: [u_brand_context_id#10399066, u_supplier_id#10399067, e_supplier_id#10399089, e_create_date#10399093, similarity#10399147, rank#10399167]
Condition : (rank#10399167 = 1)
(141) Project
Output [2]: [u_brand_context_id#10399066 AS brand_context_id#10399187, u_supplier_id#10399067 AS supplier_id#10399188]
Input [6]: [u_brand_context_id#10399066, u_supplier_id#10399067, e_supplier_id#10399089, e_create_date#10399093, similarity#10399147, rank#10399167]
(142) SortMergeJoin
Left keys [2]: [brand_context_id#10399312, supplier_id#10399313]
Right keys [2]: [brand_context_id#10399187, supplier_id#10399188]
Join type: LeftAnti
Join condition: None
(143) Project
Output [10]: [brand_context_id#10399312, supplier_id#10399313, declaring_supplier_id#10399319, supplier_type#10399318, supplier_id#10399313 AS canonical_id#10399260, 1 AS is_primary#10399261, threshold#10399322 AS configured_threshold#10399262, 0.0 AS match_percentage#10399263, 0.0 AS confidence_score#10399264, new_canonical AS match_reason#10399265]
Input [5]: [brand_context_id#10399312, supplier_id#10399313, supplier_type#10399318, declaring_supplier_id#10399319, threshold#10399322]
(144) Union
(145) AdaptiveSparkPlan
Output [10]: [brand_context_id#8983060, supplier_id#8983061, declaring_supplier_id#8983067, supplier_type#8983066, canonical_id#10397736, is_primary#10397910, configured_threshold#10397911, match_percentage#10397912, confidence_score#10397913, match_reason#10397914]
Arguments: isFinalPlan=false
(146) TableCacheQueryStage
Output [1]: [match_reason#10397914]
Arguments: 0
(147) HashAggregate [codegen id : 1]
Input [1]: [match_reason#10397914]
Keys [1]: [match_reason#10397914]
Functions [1]: [partial_count(1)]
Aggregate Attributes [1]: [count#10418144L]
Results [2]: [match_reason#10397914, count#10418145L]
(148) Exchange
Input [2]: [match_reason#10397914, count#10418145L]
Arguments: hashpartitioning(match_reason#10397914, 10), ENSURE_REQUIREMENTS, [plan_id=361669]
(149) ShuffleQueryStage
Output [2]: [match_reason#10397914, count#10418145L]
Arguments: 1
(150) AQEShuffleRead
Input [2]: [match_reason#10397914, count#10418145L]
Arguments: coalesced
(151) HashAggregate [codegen id : 2]
Input [2]: [match_reason#10397914, count#10418145L]
Keys [1]: [match_reason#10397914]
Functions [1]: [count(1)]
Aggregate Attributes [1]: [count(1)#10417990L]
Results [2]: [match_reason#10397914, count(1)#10417990L AS count#10417991L]
(152) HashAggregate
Input [1]: [match_reason#10397914]
Keys [1]: [match_reason#10397914]
Functions [1]: [partial_count(1)]
Aggregate Attributes [1]: [count#10418144L]
Results [2]: [match_reason#10397914, count#10418145L]
(153) Exchange
Input [2]: [match_reason#10397914, count#10418145L]
Arguments: hashpartitioning(match_reason#10397914, 10), ENSURE_REQUIREMENTS, [plan_id=361538]
(154) HashAggregate
Input [2]: [match_reason#10397914, count#10418145L]
Keys [1]: [match_reason#10397914]
Functions [1]: [count(1)]
Aggregate Attributes [1]: [count(1)#10417990L]
Results [2]: [match_reason#10397914, count(1)#10417990L AS count#10417991L]
(155) AdaptiveSparkPlan
Output [2]: [match_reason#10397914, count#10417991L]
Arguments: isFinalPlan=true