问题
I am trying to create a report that display potential duplicate records based on three criteria: the last 4 of an SSN, last name and DOB. I posted a question here on the issue and received an answer that I should be using a Cross Apply to unpivot the data. The query runs fast and the results look better than my original query.
The new query is below and I have added a filter to show two examples that I am seeing occur across the data:
DECLARE
@StartDate DATE = '1/1/2017',
@EndDate DATE = '3/1/2017';
WITH
CTE
AS
(
SELECT
DENSE_RANK() OVER (ORDER BY c.socialSecurityNumber) AS [SSNRanking] ,
c.id AS [CustomerID] ,
c.socialSecurityNumber AS [SSN],
c.firstName AS [FirstName] ,
c.lastName AS [LastName] ,
c.birthDate [BirthDate] ,
c.emailAddress AS [EmailAddress],
c.createDate AS [CreateDate] ,
MAX(co.orderDate) AS [LastOrderDate] ,
ca.street1 AS [Addr1] ,
ca.city AS [City] ,
ca.stateAndTerritoriesID AS [State],
ca.zipCode5 AS [Zip] ,
c2.id AS [DupCustomerID] ,
c2.socialSecurityNumber AS [DupSSN] ,
c2.firstName AS [DupFirstName] ,
c2.lastName AS [DupLastName] ,
c2.birthDate AS [DupBirthDate] ,
c2.emailAddress AS [DupEmailAddress] ,
c2.createDate AS [DupCreateDate] ,
MAX(co.orderDate) AS [DupLastOrderDate] ,
ca.street1 AS [DupAddr1] ,
ca.city AS [DupCity],
ca.stateAndTerritoriesID AS [DupState] ,
ca.zipCode5 AS [DupZip]
FROM
dbo.Customers AS [c]
INNER JOIN dbo.Customers AS [c2] ON ( SUBSTRING(c.socialSecurityNumber,6,4) = SUBSTRING(c2.socialSecurityNumber,6,4) AND c.birthDate = c2.birthDate AND c.lastName = c2.lastName AND c.id <> c2.id )
INNER JOIN dbo.CustomerAddresses AS [ca] ON c.id = ca.customerID
--INNER JOIN dbo.CustomerAddresses AS [ca2] ON ca2.customerID = c2.id
LEFT OUTER JOIN dbo.Common_Orders AS [co] ON co.customerID = c.id
WHERE
c.customerStatusTypeID <> 'M'
AND c2.customerStatusTypeID <> 'M'
AND ca.addressType = 'M'
--AND ca2.addressType = 'M'
AND c.mergedTo IS NULL
AND c2.mergedTo IS NULL
AND CAST(co.orderDate AS DATE) >= @StartDate
AND CAST(co.orderDate AS DATE) <= @EndDate
AND ( c.id = 1545229 OR c.id = 2020489 )
GROUP BY
c.id ,
c.socialSecurityNumber ,
c.firstName ,
c.lastName ,
c.birthDate ,
c.emailAddress ,
c.createDate ,
ca.street1 ,
ca.city ,
ca.stateAndTerritoriesID ,
ca.zipCode5 ,
c2.id ,
c2.socialSecurityNumber ,
c2.firstName ,
c2.lastName ,
c2.birthDate ,
c2.emailAddress ,
c2.createDate ,
ca.street1 ,
ca.city ,
ca.stateAndTerritoriesID ,
ca.zipCode5
)
SELECT CA.CustomerID,
CA.SSNRanking ,
CA.SSN ,
CA.FirstName,
CA.LastName,
CA.BirthDate,
CA.EmailAddress,
CA.CreateDate ,
CA.LastOrderDate ,
CA.Addr1,
CA.City,
CA.[State],
CA.Zip
FROM
CTE
CROSS APPLY
(
VALUES
(CTE.SSNRanking, CTE.CustomerID, CTE.SSN, CTE.FirstName, CTE.LastName, CTE.Birthdate, CTE.EmailAddress, CTE.CreateDate, CTE.LastOrderDate, CTE.Addr1, CTE.City, CTE.[State], CTE.Zip),
(CTE.SSNRanking, CTE.DupCustomerID, CTE.DupSSN, CTE.DupFirstName, CTE.DuplastName, CTE.DupBirthDate, CTE.DupEmailAddress, CTE.DupCreateDate, CTE.DupLastOrderDate, CTE.DupAddr1, CTE.DupCity, CTE.DupState, CTE.DupZip)
) AS CA (SSNRanking, CustomerID, SSN, FirstName, LastName, BirthDate, EmailAddress, CreateDate, LastOrderDate, Addr1, City, [State], Zip)
ORDER BY CAST(CA.SSN AS INT) ASC, CA.CustomerID;
And the result set looks like (Original Image):
+ ---------- + ---------- + --------- + ---------- + -------- + ---------- + ------------ + --------------------- + ----------------------- + --------------------- + ------------ + ----- + ----- +
| CustomerId | SSNRanking | SSN | FirstName | LastName | BirthDate | EmailAddress | CreateDate | LastOrderDate | Addr1 | City | State | Zip |
+ ---------- + ---------- + --------- + ---------- + -------- + ---------- + ------------ + --------------------- + ----------------------- + --------------------- + ------------ + ----- + ----- +
| 1545229 | 1 | 000000000 | Aquia Boat | SALES | 1900-01-01 | null | 2013-05-28 00:00:00.0 | 2017-01-23 11:08:30.723 | 236 Willow Landing Rd | Stafford | VA | 22554 |
| 1545229 | 1 | 000000000 | Aquia Boat | SALES | 1900-01-01 | null | 2013-05-28 00:00:00.0 | 2017-01-06 12:31:15.370 | 11963 Jefferson Ave | Newport News | VA | 23606 |
| 2020489 | 1 | 000000000 | DIXIE | SALES | 1900-01-01 | null | 2017-01-06 12:27:56.5 | 2017-01-06 12:31:15.370 | 11963 Jefferson Ave | Newport News | VA | 23606 |
| 2020489 | 1 | 000000000 | DIXIE | SALES | 1900-01-01 | null | 2017-01-06 12:27:56.5 | 2017-01-23 11:08:30.723 | 236 Willow Landing Rd | Stafford | VA | 22554 |
+ ---------- + ---------- + --------- + ---------- + -------- + ---------- + ------------ + --------------------- + ----------------------- + --------------------- + ------------ + ----- + ----- +
And I see that the last four of the SSN, last name and DOB all match. But then I noticed duplicate Customer IDs Aqua Boats has an entry with the Dixie Sales address and Dixie Sales has an entry for Aqua Boats address - which I should not have and I looked in the Customer /Customer Addresses table at the particular accounts:
SELECT c.id, c.socialSecurityNumber, c.firstName, c.lastName, c.birthDate, ca.street1
FROM dbo.Customers AS c
INNER JOIN dbo.CustomerAddresses AS [ca] ON ca.customerID = c.id
WHERE c.id IN (1545229,2020489) AND ca.addressType = 'M';
And the results are here (Original Image):
+ ---------- + -------------------- + ---------- + -------- + ---------- + --------------------- +
| id | socialSecurityNumber | firstName | lastName | birthDate | street 1 |
+ ---------- + -------------------- + ---------- + -------- + ---------- + --------------------- +
| 1545229 | 000000000 | Aquia Boat | SALES | 1900-01-01 | 236 Willow Landing Rd |
| 2020489 | 000000000 | DIXIE | SALES | 1900-01-01 | 11963 Jefferson Ave |
+ ---------- + -------------------- + ---------- + -------- + ---------- + --------------------- +
When I run the query inside the CTE, I added two additional filters:
AND c.id <> ca2.customerID
AND c2.id <> ca.customerID
And the dataset looks like I want (Original Image):
+ ---------- + ---------- + --------- + ---------- + -------- + ---------- + --------------------- + ------------ + ----- + ----- + ------------- + --------- + ------------- + ----------- + ------------- + --------------------- + ------------ + -------- + ------ +
| SSNRanking | CustomerID | SSN | FirstName | LastName | BirthDate | Addr1 | City | State | Zip | DupCustomerID | DupSSN | DupFirstName | DupLastName | DupBirthDate | DupAddr1 | DupCity | DupState | DupZip |
+ ---------- + ---------- + --------- + ---------- + -------- + ---------- + --------------------- + ------------ + ----- + ----- + ------------- + --------- + ------------- + ----------- + ------------- + --------------------- + ------------ + -------- + ------ +
| 1 | 1545229 | 000000000 | Aquia Boat | SALES | 1900-01-01 | 236 Willow Landing Rd | Stafford | VA | 22554 | 2020589 | 000000000 | DIXIE | SALES | 1900-01-01 | 236 Willow Landing Rd | Stafford | VA | 22554 |
| 1 | 2020489 | 000000000 | DIXIE | SALES | 1900-01-01 | 11963 Jefferson Ave | Newport News | VA | 23606 | 1545229 | 000000000 | AQUIA BOAT | SALES | 1900-01-01 | 11963 Jefferson Ave | Newport News | VA | 23606 |
+ ---------- + ---------- + --------- + ---------- + -------- + ---------- + --------------------- + ------------ + ----- + ----- + ------------- + --------- + ------------- + ----------- + ------------- + --------------------- + ------------ + -------- + ------ +
Can I prevent the CROSS APPLY from creating additional records for customers with mailing address assigned to another customer?
Thanks,
回答1:
i think you should throw sample data (at least 1 - 20 rows).
Or first you get your query correct then try to find duplicate records.
what do you mean by "the last 4 of an SSN, last name and DOB. " what kind of data you get ?
It appear to be very simple,
DECLARE @SampleData AS TABLE ( SSN int,lastName varchar(50),DOB datetime)
INSERT INTO @SampleData VALUES (50,'Smith','1980-02-02'),
(50,'Smith1','1980-02-02'),(50,'Smith','1980-02-02'),(50,'Smith','1980-02-02')
;With CTE as
(
select *,ROW_NUMBER()over(PARTITION by ssn,lastName,dob order by (select null))rn
from @SampleData
)
select * from cte
where rn>=4
回答2:
I kept fiddling with the report and I was able to achieve my desired results by adding another CTE and ROW_NUMBER function to remove the duplicate rows I was seeing. Seems like overkill for my desired outcome, but this runs much faster than my original query:
IF OBJECT_ID('tempdb..#DupSSN') IS NOT NULL
DROP TABLE #DupSSN;
WITH cte
AS
(
SELECT DENSE_RANK() OVER (ORDER BY c.socialSecurityNumber) AS [SSNRanking] ,
c.socialSecurityNumber AS [SSN],
RTRIM(LTRIM(SUBSTRING(c.socialSecurityNumber, 1, 3))) + '-' + RTRIM(LTRIM(SUBSTRING(c.socialSecurityNumber, 4, 2))) + '-' + RTRIM(LTRIM(SUBSTRING(c.socialSecurityNumber, 6, 4))) AS [F_SSN] ,
c.id AS [CustomerID] ,
REPLACE(LTRIM(RTRIM(c.firstName)), CHAR(13) + CHAR(10), ' ') AS [FirstName] ,
REPLACE(LTRIM(RTRIM(c.lastName)), CHAR(13) + CHAR(10), ' ') AS [LastName] ,
c.birthDate AS [BirthDate] ,
MAX(co.orderDate) AS [LastOrderDate] ,
RTRIM(LTRIM(ca.street1)) AS [Addr1] ,
RTRIM(LTRIM(ca.city)) AS [City] ,
ca.stateAndTerritoriesID AS [State] ,
ca.zipCode5 AS [Zip] ,
RTRIM(LTRIM(c.emailAddress)) AS [EmailAddress] ,
c.createDate AS [CreateDate] ,
c2.socialSecurityNumber AS [DupSSN] ,
RTRIM(LTRIM(SUBSTRING(C2.socialSecurityNumber, 1, 3))) + '-' + RTRIM(LTRIM(SUBSTRING(C2.socialSecurityNumber, 4, 2))) + '-' + RTRIM(LTRIM(SUBSTRING(C2.socialSecurityNumber, 6, 4))) AS [F_DupSSn] ,
c2.id AS [DupCustomerID] ,
REPLACE(LTRIM(RTRIM(c2.firstName)), CHAR(13) + CHAR(10), ' ') AS [DupFirstName] ,
REPLACE(LTRIM(RTRIM(c2.lastName)), CHAR(13) + CHAR(10), ' ') AS [DupLastName] ,
c2.birthDate AS [DupBirthDate] ,
MAX(co2.orderDate) AS [DupLastOrderDate] ,
RTRIM(LTRIM(ca2.street1)) AS [DupAddr1] ,
RTRIM(LTRIM(ca2.city)) AS [DupCity] ,
ca2.stateAndTerritoriesID AS [DupState] ,
ca2.zipCode5 AS [DupZip] ,
RTRIM(LTRIM(c2.emailAddress)) AS [DupEmailAddress] ,
c2.createDate AS [DupCreateDate]
FROM dbo.Customers AS [c]
INNER JOIN dbo.Customers AS [c2] ON ( SUBSTRING(c.socialSecurityNumber,6,4) = SUBSTRING(c2.socialSecurityNumber,6,4) AND c.birthDate = c2.birthDate AND c.lastName = c2.lastName AND c.id <> c2.id )
INNER JOIN dbo.CustomerAddresses AS [ca] ON c.id = ca.customerID
INNER JOIN dbo.CustomerAddresses AS [ca2] ON c2.id = ca2.customerID
INNER JOIN dbo.Common_Orders AS [co] ON co.customerID = c.id
INNER JOIN dbo.Common_Orders AS [co2] ON co2.customerID = c2.id
WHERE c.customerStatusTypeID <> 'M'
AND c2.customerStatusTypeID <> 'M'
AND ca.addressType <> 'M'
AND ca2.addressType <> 'M'
AND c.mergedTo IS NULL
AND c2.mergedTo IS NULL
AND CAST(co.orderDate AS DATE) >= '3/1/2017'
AND CAST(co.orderDate AS DATE) <= '3/31/2017'
GROUP BY c.socialSecurityNumber ,
RTRIM(LTRIM(SUBSTRING(c.socialSecurityNumber, 1, 3))) + '-' + RTRIM(LTRIM(SUBSTRING(c.socialSecurityNumber, 4, 2))) + '-' + RTRIM(LTRIM(SUBSTRING(c.socialSecurityNumber, 6, 4))) ,
c.id ,
REPLACE(LTRIM(RTRIM(c.firstName)), CHAR(13) + CHAR(10), ' ') ,
REPLACE(LTRIM(RTRIM(c.lastName)), CHAR(13) + CHAR(10), ' ') ,
c.birthDate ,
RTRIM(LTRIM(CA.street1)) ,
RTRIM(LTRIM(CA.city)) ,
CA.stateAndTerritoriesID ,
CA.zipCode5 ,
RTRIM(LTRIM(c.emailAddress)) ,
c.createDate ,
c2.socialSecurityNumber ,
RTRIM(LTRIM(SUBSTRING(C2.socialSecurityNumber, 1, 3))) + '-' + RTRIM(LTRIM(SUBSTRING(C2.socialSecurityNumber, 4, 2))) + '-' + RTRIM(LTRIM(SUBSTRING(C2.socialSecurityNumber, 6, 4))) ,
c2.id ,
REPLACE(LTRIM(RTRIM(c2.firstName)), CHAR(13) + CHAR(10), ' ') ,
REPLACE(LTRIM(RTRIM(c2.lastName)), CHAR(13) + CHAR(10), ' ') ,
c2.birthDate ,
RTRIM(LTRIM(ca2.street1)) ,
RTRIM(LTRIM(ca2.city)) ,
ca2.stateAndTerritoriesID ,
ca2.zipCode5 ,
RTRIM(LTRIM(c2.emailAddress)) ,
c2.createDate
)
-- Use a CTE to cross apply or unpivot the potential duplicates into separate rows
SELECT ca.SSNRanking ,
ca.SSN ,
ca.F_SSN ,
ca.CustomerID ,
ca.FirstName ,
ca.LastName ,
ca.BirthDate ,
ca.LastOrderDate ,
ca.Addr1 ,
ca.City ,
ca.[State] ,
ca.Zip ,
ca.EmailAddress ,
ca.CreateDate
INTO #DupSSN
FROM cte
CROSS APPLY
(
VALUES
( cte.SSNRanking, cte.SSN, cte.F_SSN, cte.CustomerID, cte.FirstName, cte.LastName, cte.BirthDate, cte.LastOrderDate, cte.Addr1, cte.City, cte.[State], cte.Zip, cte.EmailAddress, cte.CreateDate ) ,
( cte.SSNRanking, cte.DupSSN, cte.F_SSN, cte.DupCustomerID, cte.DupFirstName, cte.DupLastName, cte.DupBirthDate, cte.DupLastOrderDate,cte.DupAddr1, cte.DupCity, cte.DupState, cte.DupZip, cte.DupEmailAddress, cte.DupCreateDate )
) AS ca (SSNRanking, SSN, F_SSN, CustomerID, FirstName, LastName, BirthDate, LastOrderDate, Addr1, City, State, Zip, EmailAddress, CreateDate)
ORDER BY CAST(ca.SSN AS INT) ASC, ca.CustomerID;
-- Use CTE to create ROW_NUMBER function to eliminate duplicate records
WITH SSN_RN AS
(
SELECT SSNRanking ,
ROW_NUMBER() OVER ( PARTITION BY SSN, CustomerID, FirstName, LastName, BirthDate, LastOrderDate, Addr1, City, CreateDate ORDER BY ( SELECT NULL ) ) AS [RowNumber] ,
SSN ,
F_SSN ,
CustomerID ,
FirstName ,
LastName ,
BirthDate ,
LastOrderDate ,
Addr1 ,
City ,
State ,
Zip ,
EmailAddress ,
CreateDate
FROM #DupSSN
)
SELECT SSN_RN.SSNRanking ,
SSN_RN.RowNumber ,
SSN_RN.SSN ,
SSN_RN.F_SSN ,
SSN_RN.CustomerID ,
SSN_RN.FirstName ,
SSN_RN.LastName ,
SSN_RN.BirthDate ,
SSN_RN.LastOrderDate ,
SSN_RN.Addr1 ,
SSN_RN.City ,
SSN_RN.State ,
SSN_RN.Zip ,
SSN_RN.EmailAddress ,
SSN_RN.CreateDate
FROM SSN_RN
WHERE SSN_RN.RowNumber = 1
ORDER BY CAST(SSN_RN.SSN AS INT) ASC ,
SSN_RN.CustomerID;
DROP TABLE #DupSSN;
来源：https://stackoverflow.com/questions/43132297/cross-apply-creating-additional-records