Idempotency in API Design

Idempotent operations produce the same result regardless of how many times they're called. This is essential for reliable APIs that handle retries and network failures.

Why Idempotency Matters#

Network Failures:
- Request sent but response lost
- Client retries not knowing if it succeeded
- Without idempotency: duplicate charges, orders, etc.

With Idempotency:
- Client can safely retry
- Server detects duplicate requests
- Same result returned for repeated calls

HTTP Method Idempotency#

Naturally Idempotent:
- GET: Read operations
- PUT: Replace resource
- DELETE: Remove resource
- HEAD: Read headers
- OPTIONS: Read options

NOT Idempotent by Default:
- POST: Create resource (needs idempotency key)
- PATCH: May or may not be idempotent

Idempotency Keys#

import { Redis } from 'ioredis';
import crypto from 'crypto';

const redis = new Redis(process.env.REDIS_URL);

interface IdempotencyRecord {
  status: 'processing' | 'completed';
  response?: {
    statusCode: number;
    body: any;
  };
  createdAt: string;
}

const IDEMPOTENCY_TTL = 24 * 60 * 60; // 24 hours

async function getIdempotencyRecord(key: string): Promise<IdempotencyRecord | null> {
  const data = await redis.get(`idempotency:${key}`);
  return data ? JSON.parse(data) : null;
}

async function setIdempotencyRecord(
  key: string,
  record: IdempotencyRecord
): Promise<void> {
  await redis.setex(
    `idempotency:${key}`,
    IDEMPOTENCY_TTL,
    JSON.stringify(record)
  );
}

// Middleware
function idempotent() {
  return async (req: Request, res: Response, next: NextFunction) => {
    const idempotencyKey = req.headers['idempotency-key'] as string;

    // Only apply to POST/PATCH
    if (!['POST', 'PATCH'].includes(req.method)) {
      return next();
    }

    // Key is optional but recommended
    if (!idempotencyKey) {
      return next();
    }

    // Validate key format
    if (idempotencyKey.length > 255) {
      return res.status(400).json({
        error: 'Idempotency key too long',
      });
    }

    // Create unique key including user context
    const fullKey = `${req.user?.id || 'anonymous'}:${req.path}:${idempotencyKey}`;

    // Check for existing record
    const existing = await getIdempotencyRecord(fullKey);

    if (existing) {
      if (existing.status === 'processing') {
        // Request still in progress
        return res.status(409).json({
          error: 'Request is being processed',
          retryAfter: 1,
        });
      }

      if (existing.status === 'completed' && existing.response) {
        // Return cached response
        res.setHeader('X-Idempotent-Replay', 'true');
        return res.status(existing.response.statusCode).json(existing.response.body);
      }
    }

    // Mark as processing
    await setIdempotencyRecord(fullKey, {
      status: 'processing',
      createdAt: new Date().toISOString(),
    });

    // Capture response
    const originalJson = res.json.bind(res);
    res.json = (body) => {
      // Store completed response
      setIdempotencyRecord(fullKey, {
        status: 'completed',
        response: {
          statusCode: res.statusCode,
          body,
        },
        createdAt: new Date().toISOString(),
      }).catch((err) => {
        console.error('Failed to store idempotency record:', err);
      });

      return originalJson(body);
    };

    next();
  };
}

app.use(idempotent());

Payment Processing Example#

// Client-side: Generate idempotency key
async function createPayment(amount: number): Promise<Payment> {
  const idempotencyKey = crypto.randomUUID();

  // Store key for potential retries
  localStorage.setItem('pendingPayment', idempotencyKey);

  const response = await fetch('/api/payments', {
    method: 'POST',
    headers: {
      'Content-Type': 'application/json',
      'Idempotency-Key': idempotencyKey,
    },
    body: JSON.stringify({ amount }),
  });

  if (response.ok) {
    localStorage.removeItem('pendingPayment');
  }

  return response.json();
}

// Server-side: Process payment idempotently
app.post('/api/payments', idempotent(), async (req, res) => {
  const { amount, customerId } = req.body;

  try {
    // Create payment in database
    const payment = await prisma.payment.create({
      data: {
        id: crypto.randomUUID(),
        amount,
        customerId,
        status: 'pending',
      },
    });

    // Charge payment provider
    const charge = await stripe.charges.create({
      amount: amount * 100,
      currency: 'usd',
      customer: customerId,
      idempotencyKey: req.headers['idempotency-key'],
    });

    // Update payment status
    await prisma.payment.update({
      where: { id: payment.id },
      data: {
        status: 'completed',
        stripeChargeId: charge.id,
      },
    });

    res.status(201).json(payment);
  } catch (error) {
    // Handle specific errors
    if (error.type === 'StripeIdempotencyError') {
      // Key was reused with different parameters
      return res.status(400).json({
        error: 'Idempotency key was used with different request parameters',
      });
    }

    throw error;
  }
});

Database-Level Idempotency#

// Use unique constraints
const schema = `
  CREATE TABLE orders (
    id UUID PRIMARY KEY,
    idempotency_key VARCHAR(255) UNIQUE,
    customer_id UUID NOT NULL,
    total DECIMAL(10,2) NOT NULL,
    created_at TIMESTAMP DEFAULT NOW()
  );
`;

async function createOrder(
  idempotencyKey: string,
  orderData: CreateOrderInput
): Promise<Order> {
  try {
    // Try to create order
    const order = await prisma.order.create({
      data: {
        id: crypto.randomUUID(),
        idempotencyKey,
        ...orderData,
      },
    });

    return order;
  } catch (error) {
    // Check for unique constraint violation
    if (error.code === 'P2002' && error.meta?.target?.includes('idempotency_key')) {
      // Return existing order
      const existing = await prisma.order.findUnique({
        where: { idempotencyKey },
      });

      if (existing) {
        return existing;
      }
    }

    throw error;
  }
}

Optimistic Locking#

// Use version numbers for idempotent updates
async function updateInventory(
  productId: string,
  quantity: number,
  expectedVersion: number
): Promise<Inventory> {
  const result = await prisma.inventory.updateMany({
    where: {
      productId,
      version: expectedVersion,
    },
    data: {
      quantity,
      version: { increment: 1 },
    },
  });

  if (result.count === 0) {
    // Version mismatch - concurrent modification
    throw new ConcurrentModificationError();
  }

  return prisma.inventory.findUnique({
    where: { productId },
  });
}

// Client handles conflicts
async function decrementStock(productId: string, amount: number): Promise<void> {
  let retries = 3;

  while (retries > 0) {
    const inventory = await getInventory(productId);

    try {
      await updateInventory(
        productId,
        inventory.quantity - amount,
        inventory.version
      );
      return;
    } catch (error) {
      if (error instanceof ConcurrentModificationError && retries > 1) {
        retries--;
        continue;
      }
      throw error;
    }
  }
}

Event Processing Idempotency#

// Deduplicate webhook events
async function processWebhook(event: WebhookEvent): Promise<void> {
  const eventId = event.id;

  // Check if already processed
  const existing = await prisma.processedEvent.findUnique({
    where: { eventId },
  });

  if (existing) {
    console.log(`Event ${eventId} already processed`);
    return;
  }

  // Process in transaction
  await prisma.$transaction(async (tx) => {
    // Mark as processing
    await tx.processedEvent.create({
      data: {
        eventId,
        type: event.type,
        processedAt: new Date(),
      },
    });

    // Handle event
    switch (event.type) {
      case 'payment.succeeded':
        await handlePaymentSuccess(tx, event.data);
        break;
      case 'subscription.created':
        await handleSubscriptionCreated(tx, event.data);
        break;
    }
  });
}

// Message queue deduplication
async function processMessage(message: QueueMessage): Promise<void> {
  const messageId = message.id;

  // Use Redis for deduplication with TTL
  const key = `processed:${messageId}`;
  const wasSet = await redis.setnx(key, '1');

  if (!wasSet) {
    // Already processed
    return;
  }

  // Set expiry
  await redis.expire(key, 24 * 60 * 60);

  // Process message
  await handleMessage(message);
}

Retry Handling#

// Client-side retry with backoff
async function fetchWithRetry<T>(
  fn: () => Promise<T>,
  options: {
    maxRetries?: number;
    baseDelay?: number;
    maxDelay?: number;
  } = {}
): Promise<T> {
  const { maxRetries = 3, baseDelay = 1000, maxDelay = 10000 } = options;

  let lastError: Error;

  for (let attempt = 0; attempt <= maxRetries; attempt++) {
    try {
      return await fn();
    } catch (error) {
      lastError = error;

      // Don't retry client errors
      if (error.status >= 400 && error.status < 500 && error.status !== 429) {
        throw error;
      }

      if (attempt < maxRetries) {
        const delay = Math.min(baseDelay * Math.pow(2, attempt), maxDelay);
        const jitter = delay * 0.2 * Math.random();
        await sleep(delay + jitter);
      }
    }
  }

  throw lastError!;
}

// Usage with idempotency key
const payment = await fetchWithRetry(
  () => createPayment(amount, idempotencyKey),
  { maxRetries: 3 }
);

Best Practices#

Implementation:
✓ Generate keys client-side (UUIDs)
✓ Include user context in key
✓ Set appropriate TTL (24-48 hours)
✓ Handle "processing" state

Error Handling:
✓ Return same response for duplicates
✓ Handle parameter mismatches
✓ Log duplicate requests
✓ Clean up on failure

Performance:
✓ Use fast storage (Redis)
✓ Index idempotency columns
✓ Set TTL to auto-cleanup
✓ Consider eventual consistency

Idempotency is essential for reliable APIs. Use idempotency keys for non-idempotent operations, store responses for replay, and design your data models to handle duplicates gracefully. This enables safe client retries and improves system reliability.