diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..544d21e --- /dev/null +++ b/.gitignore @@ -0,0 +1,38 @@ +# Dependencies +node_modules/ +package-lock.json +yarn.lock + +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Coverage directory +coverage/ +.nyc_output/ + +# Editor directories and files +.vscode/ +.idea/ +*.swp +*.swo +*~ +.DS_Store + +# Temporary files +tmp/ +temp/ +*.tmp + +# Build output +dist/ +build/ diff --git a/README.md b/README.md index 795eed4..3c5b343 100644 --- a/README.md +++ b/README.md @@ -1 +1,159 @@ -# testRepo \ No newline at end of file +# testRepo + +Job Handler with Deduplication Support + +## Overview + +This repository provides a `handleSendToJob` function that handles sending jobs to a processing queue with built-in deduplication support. It prevents duplicate jobs from being processed by tracking job IDs. + +## Features + +- **Deduplication**: Automatically prevents duplicate jobs from being sent based on job ID +- **Error Handling**: Comprehensive error handling for invalid inputs and send failures +- **Tracking**: Utilities to check processed job count and status +- **Simple API**: Easy-to-use function interface with clear return values + +## Installation + +```bash +npm install +``` + +## Usage + +### Basic Usage + +```javascript +const { handleSendToJob } = require('./jobHandler'); + +// Define a function that sends jobs to your queue/API +function sendToQueue(job) { + // Your implementation here (e.g., HTTP request, queue push, etc.) + console.log('Sending job:', job.id); + return { success: true }; +} + +// Send a job +const result = handleSendToJob( + { id: 'job123', data: { task: 'process-data' } }, + sendToQueue +); + +console.log(result); +// Output: { success: true, status: 'sent', message: 'Job job123 sent successfully', jobId: 'job123', result: {...} } +``` + +### Deduplication in Action + +```javascript +const { handleSendToJob } = require('./jobHandler'); + +function sendToQueue(job) { + console.log('Processing job:', job.id); + return { processed: true }; +} + +// First attempt - succeeds +const result1 = handleSendToJob({ id: 'job123', data: 'test' }, sendToQueue); +console.log(result1.success); // true +console.log(result1.status); // 'sent' + +// Second attempt with same ID - fails (duplicate) +const result2 = handleSendToJob({ id: 'job123', data: 'test' }, sendToQueue); +console.log(result2.success); // false +console.log(result2.status); // 'duplicate' +console.log(result2.message); // 'Job with ID job123 has already been processed' +``` + +### Utility Functions + +```javascript +const { + handleSendToJob, + clearProcessedJobs, + getProcessedJobCount, + hasJobBeenProcessed, +} = require('./jobHandler'); + +// Check if a job has been processed +if (hasJobBeenProcessed('job123')) { + console.log('Job already processed'); +} + +// Get count of processed jobs +console.log('Processed jobs:', getProcessedJobCount()); + +// Clear the deduplication cache (useful for testing or reset) +clearProcessedJobs(); +``` + +## API Reference + +### `handleSendToJob(job, sendFunction)` + +Sends a job to the processing queue with deduplication. + +**Parameters:** +- `job` (Object): The job object to process + - `id` (string, required): Unique identifier for the job + - `data` (any): Job data/payload +- `sendFunction` (Function): Function to send the job (e.g., to queue/API) + +**Returns:** +- Object with the following properties: + - `success` (boolean): Whether the operation succeeded + - `status` (string): Status code ('sent', 'duplicate', or 'error') + - `message` (string): Human-readable message + - `jobId` (string): The job ID + - `result` (any): Result from the send function (if successful) + - `error` (Error): Error object (if failed) + +### `clearProcessedJobs()` + +Clears the deduplication cache. Useful for testing or when you want to reset tracking. + +### `getProcessedJobCount()` + +Returns the number of jobs that have been processed. + +**Returns:** `number` + +### `hasJobBeenProcessed(jobId)` + +Checks if a job ID has been processed. + +**Parameters:** +- `jobId` (string): The job ID to check + +**Returns:** `boolean` + +## Testing + +Run the test suite: + +```bash +npm test +``` + +## Implementation Details + +The deduplication mechanism uses a `Set` to track processed job IDs in memory. This provides: +- O(1) lookup time for checking duplicates +- Efficient memory usage +- Simple and reliable deduplication + +**Note:** The deduplication cache is stored in memory and will be cleared when the process restarts. For persistent deduplication across restarts, consider using a database or external cache (Redis, etc.). + +## Error Handling + +The function handles various error scenarios: +- Invalid job object (null, undefined, or not an object) +- Missing job ID +- Missing or invalid send function +- Exceptions thrown by the send function + +All errors return a structured response with `success: false` and descriptive error messages. + +## License + +ISC \ No newline at end of file diff --git a/demo.js b/demo.js new file mode 100644 index 0000000..3c97daa --- /dev/null +++ b/demo.js @@ -0,0 +1,65 @@ +/** + * Visual demonstration of deduplication in action + */ + +const { + handleSendToJob, + clearProcessedJobs, + getProcessedJobCount, +} = require('./jobHandler'); + +// Mock queue sender +function sendToQueue(job) { + return { queued: true, timestamp: Date.now() }; +} + +console.log('\n╔════════════════════════════════════════════════════════════╗'); +console.log('║ handleSendToJob - Deduplication Demonstration ║'); +console.log('╚════════════════════════════════════════════════════════════╝\n'); + +clearProcessedJobs(); + +// Simulate receiving jobs (some duplicates) +const incomingJobs = [ + { id: 'order-101', data: { amount: 150.00 } }, + { id: 'order-102', data: { amount: 200.00 } }, + { id: 'order-101', data: { amount: 150.00 } }, // DUPLICATE + { id: 'order-103', data: { amount: 75.50 } }, + { id: 'order-102', data: { amount: 200.00 } }, // DUPLICATE + { id: 'order-104', data: { amount: 300.00 } }, + { id: 'order-101', data: { amount: 150.00 } }, // DUPLICATE + { id: 'order-105', data: { amount: 125.00 } }, +]; + +console.log('Processing incoming jobs...\n'); + +const stats = { + sent: 0, + duplicates: 0, + errors: 0, +}; + +incomingJobs.forEach((job, index) => { + const result = handleSendToJob(job, sendToQueue); + + const icon = result.success ? '✓' : '✗'; + const statusColor = result.status === 'sent' ? '🟢' : + result.status === 'duplicate' ? '🟡' : '🔴'; + + console.log(`${index + 1}. ${statusColor} Job ${job.id}: ${icon} ${result.status.toUpperCase()}`); + + if (result.status === 'sent') stats.sent++; + else if (result.status === 'duplicate') stats.duplicates++; + else stats.errors++; +}); + +console.log('\n' + '─'.repeat(60)); +console.log('\n📊 Summary:'); +console.log(` Total jobs attempted: ${incomingJobs.length}`); +console.log(` Successfully sent: ${stats.sent}`); +console.log(` Duplicates blocked: ${stats.duplicates}`); +console.log(` Errors: ${stats.errors}`); +console.log(` Unique jobs tracked: ${getProcessedJobCount()}`); + +console.log('\n💡 Deduplication prevented ' + stats.duplicates + ' duplicate job(s) from being processed!'); +console.log('\n' + '═'.repeat(60) + '\n'); diff --git a/example.js b/example.js new file mode 100644 index 0000000..4e382b9 --- /dev/null +++ b/example.js @@ -0,0 +1,135 @@ +/** + * Example usage of handleSendToJob with deduplication + */ + +const { + handleSendToJob, + clearProcessedJobs, + getProcessedJobCount, + hasJobBeenProcessed, +} = require('./jobHandler'); + +// Mock function that simulates sending jobs to a queue/API +function mockJobQueue(job) { + console.log(` → Sending job ${job.id} to queue...`); + // Simulate some processing + return { + queuePosition: Math.floor(Math.random() * 100), + estimatedTime: Math.floor(Math.random() * 60), + timestamp: new Date().toISOString(), + }; +} + +console.log('\n=== Example: handleSendToJob with Deduplication ===\n'); + +// Clear any previous state +clearProcessedJobs(); + +// Example 1: Send a new job +console.log('Example 1: Sending a new job'); +const result1 = handleSendToJob( + { + id: 'task-001', + data: { + type: 'data-processing', + payload: { records: 1000 }, + }, + }, + mockJobQueue +); +console.log('Result:', { + success: result1.success, + status: result1.status, + message: result1.message, +}); +console.log(''); + +// Example 2: Try to send the same job again (should be deduplicated) +console.log('Example 2: Attempting to send duplicate job'); +const result2 = handleSendToJob( + { + id: 'task-001', + data: { + type: 'data-processing', + payload: { records: 1000 }, + }, + }, + mockJobQueue +); +console.log('Result:', { + success: result2.success, + status: result2.status, + message: result2.message, +}); +console.log(''); + +// Example 3: Send different jobs +console.log('Example 3: Sending multiple different jobs'); +const jobs = [ + { id: 'task-002', data: { type: 'email', recipient: 'user@example.com' } }, + { id: 'task-003', data: { type: 'notification', message: 'Hello' } }, + { id: 'task-004', data: { type: 'backup', target: 'database' } }, +]; + +jobs.forEach((job) => { + const result = handleSendToJob(job, mockJobQueue); + console.log(` Job ${job.id}: ${result.status}`); +}); +console.log(''); + +// Example 4: Check processed jobs +console.log('Example 4: Checking processed jobs'); +console.log(`Total processed jobs: ${getProcessedJobCount()}`); +console.log(`Has task-001 been processed? ${hasJobBeenProcessed('task-001')}`); +console.log(`Has task-005 been processed? ${hasJobBeenProcessed('task-005')}`); +console.log(''); + +// Example 5: Handling errors +console.log('Example 5: Handling error scenarios'); + +// Missing job ID +const result5a = handleSendToJob({ data: 'test' }, mockJobQueue); +console.log(`Missing ID: ${result5a.message}`); + +// Invalid job object +const result5b = handleSendToJob(null, mockJobQueue); +console.log(`Null job: ${result5b.message}`); + +// Send function that throws error +const errorFunc = () => { + throw new Error('Connection timeout'); +}; +const result5c = handleSendToJob({ id: 'task-error', data: 'test' }, errorFunc); +console.log(`Send error: ${result5c.message}`); +console.log(''); + +// Example 6: Batch processing with deduplication +console.log('Example 6: Batch processing with duplicate detection'); +const batchJobs = [ + { id: 'batch-001', data: 'item1' }, + { id: 'batch-002', data: 'item2' }, + { id: 'batch-001', data: 'item1-duplicate' }, // Duplicate + { id: 'batch-003', data: 'item3' }, + { id: 'batch-002', data: 'item2-duplicate' }, // Duplicate + { id: 'batch-004', data: 'item4' }, +]; + +let sent = 0; +let duplicates = 0; +let errors = 0; + +batchJobs.forEach((job) => { + const result = handleSendToJob(job, mockJobQueue); + if (result.status === 'sent') sent++; + else if (result.status === 'duplicate') duplicates++; + else if (result.status === 'error') errors++; +}); + +console.log(`Batch Results: + - Successfully sent: ${sent} + - Duplicates detected: ${duplicates} + - Errors: ${errors} + - Total attempted: ${batchJobs.length} +`); + +console.log('=== Examples Complete ===\n'); diff --git a/jobHandler.js b/jobHandler.js new file mode 100644 index 0000000..ff376a7 --- /dev/null +++ b/jobHandler.js @@ -0,0 +1,109 @@ +/** + * Job Handler Module + * Handles sending jobs to a processing queue with deduplication support + */ + +// Store for tracking processed job IDs to prevent duplicates +const processedJobs = new Set(); + +/** + * Handles sending a job to the job processing queue with deduplication + * @param {Object} job - The job object to process + * @param {string} job.id - Unique identifier for the job + * @param {*} job.data - Job data/payload + * @param {Function} sendFunction - Function to actually send the job (e.g., to queue/API) + * @returns {Object} Result object with status and message + */ +function handleSendToJob(job, sendFunction) { + // Validate input + if (!job || typeof job !== 'object') { + return { + success: false, + status: 'error', + message: 'Invalid job object provided', + }; + } + + if (!job.id) { + return { + success: false, + status: 'error', + message: 'Job ID is required for deduplication', + }; + } + + // Check if job has already been processed (deduplication) + if (processedJobs.has(job.id)) { + return { + success: false, + status: 'duplicate', + message: `Job with ID ${job.id} has already been processed`, + jobId: job.id, + }; + } + + // Validate send function + if (!sendFunction || typeof sendFunction !== 'function') { + return { + success: false, + status: 'error', + message: 'Valid send function is required', + }; + } + + try { + // Send the job using the provided function + const result = sendFunction(job); + + // Mark job as processed for deduplication + processedJobs.add(job.id); + + return { + success: true, + status: 'sent', + message: `Job ${job.id} sent successfully`, + jobId: job.id, + result: result, + }; + } catch (error) { + return { + success: false, + status: 'error', + message: `Failed to send job: ${error.message}`, + jobId: job.id, + error: error, + }; + } +} + +/** + * Clears the deduplication cache + * Useful for testing or when you want to reset the tracking + */ +function clearProcessedJobs() { + processedJobs.clear(); +} + +/** + * Gets the count of processed jobs + * @returns {number} Number of jobs that have been processed + */ +function getProcessedJobCount() { + return processedJobs.size; +} + +/** + * Checks if a job ID has been processed + * @param {string} jobId - The job ID to check + * @returns {boolean} True if the job has been processed + */ +function hasJobBeenProcessed(jobId) { + return processedJobs.has(jobId); +} + +module.exports = { + handleSendToJob, + clearProcessedJobs, + getProcessedJobCount, + hasJobBeenProcessed, +}; diff --git a/package.json b/package.json new file mode 100644 index 0000000..2392d70 --- /dev/null +++ b/package.json @@ -0,0 +1,16 @@ +{ + "name": "testRepo", + "version": "1.0.0", + "description": "Job handler with deduplication support", + "main": "jobHandler.js", + "scripts": { + "test": "node test.js" + }, + "keywords": [ + "job", + "deduplication", + "queue" + ], + "author": "", + "license": "ISC" +} diff --git a/test.js b/test.js new file mode 100644 index 0000000..f05eee6 --- /dev/null +++ b/test.js @@ -0,0 +1,158 @@ +/** + * Test suite for handleSendToJob function + */ + +const { + handleSendToJob, + clearProcessedJobs, + getProcessedJobCount, + hasJobBeenProcessed, +} = require('./jobHandler'); + +// Simple test runner +let testsPassed = 0; +let testsFailed = 0; + +function assert(condition, message) { + if (condition) { + console.log(`✓ ${message}`); + testsPassed++; + } else { + console.error(`✗ ${message}`); + testsFailed++; + } +} + +function assertEqual(actual, expected, message) { + if (actual === expected) { + console.log(`✓ ${message}`); + testsPassed++; + } else { + console.error(`✗ ${message}`); + console.error(` Expected: ${expected}`); + console.error(` Actual: ${actual}`); + testsFailed++; + } +} + +// Mock send function +const mockSendFunction = (job) => { + return { sent: true, jobId: job.id }; +}; + +console.log('\n=== Testing handleSendToJob with Deduplication ===\n'); + +// Test 1: Valid job should be sent successfully +console.log('Test 1: Valid job should be sent successfully'); +clearProcessedJobs(); +const result1 = handleSendToJob({ id: 'job1', data: 'test' }, mockSendFunction); +assert(result1.success === true, 'Job should be sent successfully'); +assert(result1.status === 'sent', 'Status should be "sent"'); +assert(result1.jobId === 'job1', 'Job ID should be "job1"'); + +// Test 2: Duplicate job should be rejected +console.log('\nTest 2: Duplicate job should be rejected'); +const result2 = handleSendToJob({ id: 'job1', data: 'test' }, mockSendFunction); +assert(result2.success === false, 'Duplicate job should fail'); +assert(result2.status === 'duplicate', 'Status should be "duplicate"'); +assert(result2.message.includes('already been processed'), 'Message should indicate duplicate'); + +// Test 3: Different job should be sent successfully +console.log('\nTest 3: Different job should be sent successfully'); +const result3 = handleSendToJob({ id: 'job2', data: 'test2' }, mockSendFunction); +assert(result3.success === true, 'Different job should be sent successfully'); +assert(result3.status === 'sent', 'Status should be "sent"'); + +// Test 4: Invalid job (missing ID) should be rejected +console.log('\nTest 4: Job without ID should be rejected'); +const result4 = handleSendToJob({ data: 'test' }, mockSendFunction); +assert(result4.success === false, 'Job without ID should fail'); +assert(result4.message.includes('Job ID is required'), 'Message should indicate missing ID'); + +// Test 5: Invalid job (null) should be rejected +console.log('\nTest 5: Null job should be rejected'); +const result5 = handleSendToJob(null, mockSendFunction); +assert(result5.success === false, 'Null job should fail'); +assert(result5.message.includes('Invalid job object'), 'Message should indicate invalid job'); + +// Test 6: Missing send function should be rejected +console.log('\nTest 6: Missing send function should be rejected'); +const result6 = handleSendToJob({ id: 'job3', data: 'test' }, null); +assert(result6.success === false, 'Missing send function should fail'); +assert(result6.message.includes('send function is required'), 'Message should indicate missing send function'); + +// Test 7: Error in send function should be caught +console.log('\nTest 7: Error in send function should be caught'); +clearProcessedJobs(); +const errorSendFunction = () => { + throw new Error('Send failed'); +}; +const result7 = handleSendToJob({ id: 'job4', data: 'test' }, errorSendFunction); +assert(result7.success === false, 'Job with error should fail'); +assert(result7.status === 'error', 'Status should be "error"'); +assert(result7.message.includes('Failed to send job'), 'Message should indicate send failure'); + +// Test 8: clearProcessedJobs should reset deduplication +console.log('\nTest 8: clearProcessedJobs should reset deduplication'); +clearProcessedJobs(); +const result8a = handleSendToJob({ id: 'job5', data: 'test' }, mockSendFunction); +assert(result8a.success === true, 'First send should succeed'); +clearProcessedJobs(); +const result8b = handleSendToJob({ id: 'job5', data: 'test' }, mockSendFunction); +assert(result8b.success === true, 'After clear, same job should succeed again'); + +// Test 9: getProcessedJobCount should return correct count +console.log('\nTest 9: getProcessedJobCount should return correct count'); +clearProcessedJobs(); +assertEqual(getProcessedJobCount(), 0, 'Initial count should be 0'); +handleSendToJob({ id: 'job6', data: 'test' }, mockSendFunction); +assertEqual(getProcessedJobCount(), 1, 'Count should be 1 after one job'); +handleSendToJob({ id: 'job7', data: 'test' }, mockSendFunction); +assertEqual(getProcessedJobCount(), 2, 'Count should be 2 after two jobs'); +handleSendToJob({ id: 'job6', data: 'test' }, mockSendFunction); // Duplicate +assertEqual(getProcessedJobCount(), 2, 'Count should stay 2 after duplicate'); + +// Test 10: hasJobBeenProcessed should work correctly +console.log('\nTest 10: hasJobBeenProcessed should work correctly'); +clearProcessedJobs(); +assert(hasJobBeenProcessed('job8') === false, 'Job8 should not be processed initially'); +handleSendToJob({ id: 'job8', data: 'test' }, mockSendFunction); +assert(hasJobBeenProcessed('job8') === true, 'Job8 should be marked as processed'); +assert(hasJobBeenProcessed('job9') === false, 'Job9 should not be processed'); + +// Test 11: Multiple jobs in sequence +console.log('\nTest 11: Multiple jobs in sequence'); +clearProcessedJobs(); +const jobs = [ + { id: 'seq1', data: 'data1' }, + { id: 'seq2', data: 'data2' }, + { id: 'seq3', data: 'data3' }, + { id: 'seq1', data: 'data1' }, // Duplicate + { id: 'seq4', data: 'data4' }, +]; + +let successCount = 0; +let duplicateCount = 0; + +jobs.forEach((job) => { + const result = handleSendToJob(job, mockSendFunction); + if (result.success) successCount++; + if (result.status === 'duplicate') duplicateCount++; +}); + +assertEqual(successCount, 4, 'Should successfully send 4 unique jobs'); +assertEqual(duplicateCount, 1, 'Should detect 1 duplicate'); +assertEqual(getProcessedJobCount(), 4, 'Should track 4 unique jobs'); + +// Summary +console.log('\n=== Test Summary ==='); +console.log(`Tests Passed: ${testsPassed}`); +console.log(`Tests Failed: ${testsFailed}`); + +if (testsFailed === 0) { + console.log('\n✓ All tests passed!'); + process.exit(0); +} else { + console.log(`\n✗ ${testsFailed} test(s) failed`); + process.exit(1); +}