feat: add transaction deduplication feature and enhance filtering options in settings and transactions pages

services/transaction.service.ts

@@ -8,25 +8,56 @@ export interface CreateManyResult {
 
 export const transactionService = {
   async createMany(transactions: Transaction[]): Promise<CreateManyResult> {
-    // Filter out duplicates based on fitId (business rule)
-    const existingTransactions = await prisma.transaction.findMany({
+    // Get unique account IDs
+    const accountIds = [...new Set(transactions.map((t) => t.accountId))];
+    
+    // Check for existing transactions by fitId
+    const existingByFitId = await prisma.transaction.findMany({
       where: {
-        accountId: { in: transactions.map((t) => t.accountId) },
+        accountId: { in: accountIds },
         fitId: { in: transactions.map((t) => t.fitId) },
       },
       select: {
         accountId: true,
         fitId: true,
+        date: true,
+        amount: true,
+        description: true,
       },
     });
 
-    const existingSet = new Set(
-      existingTransactions.map((t) => `${t.accountId}-${t.fitId}`),
+    // Get all existing transactions for these accounts to check duplicates by criteria
+    const allExistingTransactions = await prisma.transaction.findMany({
+      where: {
+        accountId: { in: accountIds },
+      },
+      select: {
+        accountId: true,
+        date: true,
+        amount: true,
+        description: true,
+      },
+    });
+
+    // Create sets for fast lookup
+    const existingFitIdSet = new Set(
+      existingByFitId.map((t) => `${t.accountId}-${t.fitId}`),
+    );
+    
+    // Create set for duplicates by amount + date + description
+    const existingCriteriaSet = new Set(
+      allExistingTransactions.map((t) => 
+        `${t.accountId}-${t.date}-${t.amount}-${t.description}`
+      ),
     );
 
-    const newTransactions = transactions.filter(
-      (t) => !existingSet.has(`${t.accountId}-${t.fitId}`),
-    );
+    // Filter out duplicates based on fitId OR (amount + date + description)
+    const newTransactions = transactions.filter((t) => {
+      const fitIdKey = `${t.accountId}-${t.fitId}`;
+      const criteriaKey = `${t.accountId}-${t.date}-${t.amount}-${t.description}`;
+      
+      return !existingFitIdSet.has(fitIdKey) && !existingCriteriaSet.has(criteriaKey);
+    });
 
     if (newTransactions.length === 0) {
       return { count: 0, transactions: [] };
@@ -90,6 +121,65 @@ export const transactionService = {
       where: { id },
     });
   },
+
+  async deduplicate(): Promise<{ deletedCount: number; duplicatesFound: number }> {
+    // Get all transactions grouped by account
+    const allTransactions = await prisma.transaction.findMany({
+      orderBy: [
+        { accountId: "asc" },
+        { date: "asc" },
+        { id: "asc" }, // Keep the oldest transaction (first created)
+      ],
+      select: {
+        id: true,
+        accountId: true,
+        date: true,
+        amount: true,
+        description: true,
+      },
+    });
+
+    // Group by account for efficient processing
+    const transactionsByAccount = new Map<string, typeof allTransactions>();
+    for (const transaction of allTransactions) {
+      if (!transactionsByAccount.has(transaction.accountId)) {
+        transactionsByAccount.set(transaction.accountId, []);
+      }
+      transactionsByAccount.get(transaction.accountId)!.push(transaction);
+    }
+
+    const duplicatesToDelete: string[] = [];
+    const seenKeys = new Set<string>();
+
+    // For each account, find duplicates
+    for (const [accountId, transactions] of transactionsByAccount.entries()) {
+      for (const transaction of transactions) {
+        const key = `${accountId}-${transaction.date}-${transaction.amount}-${transaction.description}`;
+        
+        if (seenKeys.has(key)) {
+          // This is a duplicate, mark for deletion
+          duplicatesToDelete.push(transaction.id);
+        } else {
+          // First occurrence, keep it
+          seenKeys.add(key);
+        }
+      }
+    }
+
+    // Delete duplicates
+    if (duplicatesToDelete.length > 0) {
+      await prisma.transaction.deleteMany({
+        where: {
+          id: { in: duplicatesToDelete },
+        },
+      });
+    }
+
+    return {
+      deletedCount: duplicatesToDelete.length,
+      duplicatesFound: duplicatesToDelete.length,
+    };
+  },
 };