Node.js Zlib Compression Guide

The zlib module provides compression using Gzip and Deflate/Inflate algorithms. Here's how to use it.

Basic Compression#

const zlib = require('zlib');
const { promisify } = require('util');

// Promisified versions
const gzip = promisify(zlib.gzip);
const gunzip = promisify(zlib.gunzip);
const deflate = promisify(zlib.deflate);
const inflate = promisify(zlib.inflate);

// Compress string
async function compressString(input) {
  const buffer = Buffer.from(input, 'utf8');
  const compressed = await gzip(buffer);
  console.log(`Original: ${buffer.length} bytes`);
  console.log(`Compressed: ${compressed.length} bytes`);
  return compressed;
}

// Decompress
async function decompressString(compressed) {
  const decompressed = await gunzip(compressed);
  return decompressed.toString('utf8');
}

// Usage
async function example() {
  const text = 'Hello, World! '.repeat(100);
  const compressed = await compressString(text);
  const original = await decompressString(compressed);
  console.log(original === text); // true
}

Sync Methods#

const zlib = require('zlib');

// Synchronous compression
const input = 'Data to compress';
const buffer = Buffer.from(input);

// Gzip
const gzipped = zlib.gzipSync(buffer);
const gunzipped = zlib.gunzipSync(gzipped);

// Deflate
const deflated = zlib.deflateSync(buffer);
const inflated = zlib.inflateSync(deflated);

// Brotli (Node 11.7+)
const brotli = zlib.brotliCompressSync(buffer);
const unbrotli = zlib.brotliDecompressSync(brotli);

console.log(gunzipped.toString()); // 'Data to compress'
console.log(inflated.toString()); // 'Data to compress'
console.log(unbrotli.toString()); // 'Data to compress'

Stream Compression#

const zlib = require('zlib');
const fs = require('fs');
const { pipeline } = require('stream/promises');

// Compress file
async function compressFile(inputPath, outputPath) {
  const gzip = zlib.createGzip();
  const source = fs.createReadStream(inputPath);
  const destination = fs.createWriteStream(outputPath);

  await pipeline(source, gzip, destination);
  console.log(`Compressed ${inputPath} to ${outputPath}`);
}

// Decompress file
async function decompressFile(inputPath, outputPath) {
  const gunzip = zlib.createGunzip();
  const source = fs.createReadStream(inputPath);
  const destination = fs.createWriteStream(outputPath);

  await pipeline(source, gunzip, destination);
  console.log(`Decompressed ${inputPath} to ${outputPath}`);
}

// Usage
await compressFile('large-file.txt', 'large-file.txt.gz');
await decompressFile('large-file.txt.gz', 'restored-file.txt');

HTTP Compression#

const http = require('http');
const zlib = require('zlib');
const fs = require('fs');

const server = http.createServer((req, res) => {
  const acceptEncoding = req.headers['accept-encoding'] || '';

  const raw = fs.createReadStream('index.html');

  // Check client support
  if (acceptEncoding.includes('br')) {
    res.writeHead(200, { 'Content-Encoding': 'br' });
    raw.pipe(zlib.createBrotliCompress()).pipe(res);
  } else if (acceptEncoding.includes('gzip')) {
    res.writeHead(200, { 'Content-Encoding': 'gzip' });
    raw.pipe(zlib.createGzip()).pipe(res);
  } else if (acceptEncoding.includes('deflate')) {
    res.writeHead(200, { 'Content-Encoding': 'deflate' });
    raw.pipe(zlib.createDeflate()).pipe(res);
  } else {
    res.writeHead(200);
    raw.pipe(res);
  }
});

server.listen(3000);

// Client request with compression
async function fetchCompressed(url) {
  const response = await fetch(url, {
    headers: {
      'Accept-Encoding': 'gzip, deflate, br',
    },
  });

  // fetch() automatically decompresses
  return response.text();
}

Compression Options#

const zlib = require('zlib');

// Gzip with options
const gzip = zlib.createGzip({
  level: 9, // 0-9, higher = more compression
  memLevel: 8, // 1-9, memory usage
  strategy: zlib.constants.Z_DEFAULT_STRATEGY,
});

// Compression levels
const levels = {
  Z_NO_COMPRESSION: 0,
  Z_BEST_SPEED: 1,
  Z_BEST_COMPRESSION: 9,
  Z_DEFAULT_COMPRESSION: -1,
};

// Strategies
const strategies = {
  Z_FILTERED: zlib.constants.Z_FILTERED,
  Z_HUFFMAN_ONLY: zlib.constants.Z_HUFFMAN_ONLY,
  Z_RLE: zlib.constants.Z_RLE,
  Z_FIXED: zlib.constants.Z_FIXED,
  Z_DEFAULT_STRATEGY: zlib.constants.Z_DEFAULT_STRATEGY,
};

// Brotli options
const brotli = zlib.createBrotliCompress({
  params: {
    [zlib.constants.BROTLI_PARAM_QUALITY]: 11, // 0-11
    [zlib.constants.BROTLI_PARAM_SIZE_HINT]: 1000,
  },
});

// Compare compression methods
async function compareCompression(data) {
  const buffer = Buffer.from(data);

  const gzipped = zlib.gzipSync(buffer, { level: 9 });
  const deflated = zlib.deflateSync(buffer, { level: 9 });
  const brotli = zlib.brotliCompressSync(buffer);

  console.log(`Original: ${buffer.length} bytes`);
  console.log(`Gzip: ${gzipped.length} bytes`);
  console.log(`Deflate: ${deflated.length} bytes`);
  console.log(`Brotli: ${brotli.length} bytes`);
}

Error Handling#

const zlib = require('zlib');
const { pipeline } = require('stream/promises');
const fs = require('fs');

// Stream error handling
async function safeCompress(inputPath, outputPath) {
  try {
    await pipeline(
      fs.createReadStream(inputPath),
      zlib.createGzip(),
      fs.createWriteStream(outputPath)
    );
    console.log('Compression complete');
  } catch (error) {
    console.error('Compression failed:', error.message);

    // Clean up partial output
    try {
      fs.unlinkSync(outputPath);
    } catch {}
  }
}

// Callback error handling
function compressWithCallback(data, callback) {
  zlib.gzip(data, (err, result) => {
    if (err) {
      callback(err, null);
      return;
    }
    callback(null, result);
  });
}

// Handle corrupted data
async function safeDecompress(compressed) {
  try {
    const gunzip = require('util').promisify(zlib.gunzip);
    return await gunzip(compressed);
  } catch (error) {
    if (error.code === 'Z_DATA_ERROR') {
      throw new Error('Invalid or corrupted compressed data');
    }
    throw error;
  }
}

Chunked Processing#

const zlib = require('zlib');
const { Transform } = require('stream');

// Process compressed data in chunks
class ChunkProcessor extends Transform {
  constructor(options) {
    super(options);
    this.chunks = [];
  }

  _transform(chunk, encoding, callback) {
    // Process each chunk
    console.log(`Processing chunk: ${chunk.length} bytes`);
    this.chunks.push(chunk);
    callback(null, chunk);
  }

  _flush(callback) {
    console.log(`Total chunks: ${this.chunks.length}`);
    callback();
  }
}

// Usage with compression
async function processCompressed(inputPath) {
  const { pipeline } = require('stream/promises');

  await pipeline(
    fs.createReadStream(inputPath),
    zlib.createGunzip(),
    new ChunkProcessor(),
    fs.createWriteStream('/dev/null')
  );
}

// Memory-efficient large file compression
async function compressLargeFile(inputPath, outputPath) {
  const gzip = zlib.createGzip({
    chunkSize: 64 * 1024, // 64KB chunks
    level: 6,
  });

  let bytesRead = 0;
  const input = fs.createReadStream(inputPath, {
    highWaterMark: 64 * 1024,
  });

  input.on('data', (chunk) => {
    bytesRead += chunk.length;
    process.stdout.write(`\rProcessed: ${bytesRead} bytes`);
  });

  await pipeline(input, gzip, fs.createWriteStream(outputPath));
  console.log('\nDone');
}

JSON Compression#

const zlib = require('zlib');
const { promisify } = require('util');

const gzip = promisify(zlib.gzip);
const gunzip = promisify(zlib.gunzip);

// Compress JSON
async function compressJSON(data) {
  const json = JSON.stringify(data);
  const compressed = await gzip(Buffer.from(json));
  return compressed.toString('base64');
}

// Decompress JSON
async function decompressJSON(compressed) {
  const buffer = Buffer.from(compressed, 'base64');
  const decompressed = await gunzip(buffer);
  return JSON.parse(decompressed.toString());
}

// Usage
async function example() {
  const data = {
    users: Array(100)
      .fill(null)
      .map((_, i) => ({
        id: i,
        name: `User ${i}`,
        email: `user${i}@example.com`,
      })),
  };

  const compressed = await compressJSON(data);
  console.log(`Compressed length: ${compressed.length}`);

  const restored = await decompressJSON(compressed);
  console.log(`Users: ${restored.users.length}`);
}

// Cache with compression
class CompressedCache {
  constructor() {
    this.cache = new Map();
  }

  async set(key, value) {
    const compressed = await compressJSON(value);
    this.cache.set(key, compressed);
  }

  async get(key) {
    const compressed = this.cache.get(key);
    if (!compressed) return null;
    return decompressJSON(compressed);
  }

  has(key) {
    return this.cache.has(key);
  }
}

Archive Operations#

const zlib = require('zlib');
const tar = require('tar'); // npm package
const fs = require('fs');

// Create tar.gz archive
async function createArchive(sourceDir, outputPath) {
  await tar.create(
    {
      gzip: true,
      file: outputPath,
    },
    [sourceDir]
  );
  console.log(`Created archive: ${outputPath}`);
}

// Extract tar.gz archive
async function extractArchive(archivePath, destDir) {
  await tar.extract({
    file: archivePath,
    cwd: destDir,
  });
  console.log(`Extracted to: ${destDir}`);
}

// Manual tar.gz with streams
const { pipeline } = require('stream/promises');

async function createGzipArchive(inputPath, outputPath) {
  await pipeline(
    tar.create({ gzip: false }, [inputPath]),
    zlib.createGzip(),
    fs.createWriteStream(outputPath)
  );
}

// List archive contents
async function listArchive(archivePath) {
  const files = [];

  await tar.list({
    file: archivePath,
    onentry: (entry) => {
      files.push({
        path: entry.path,
        size: entry.size,
        type: entry.type,
      });
    },
  });

  return files;
}

Performance Optimization#

const zlib = require('zlib');
const { Worker, isMainThread, parentPort } = require('worker_threads');

// Worker thread compression
if (isMainThread) {
  async function compressInWorker(data) {
    return new Promise((resolve, reject) => {
      const worker = new Worker(__filename);
      worker.postMessage(data);
      worker.on('message', resolve);
      worker.on('error', reject);
    });
  }
} else {
  parentPort.on('message', async (data) => {
    const compressed = zlib.gzipSync(data);
    parentPort.postMessage(compressed);
  });
}

// Compression pool
class CompressionPool {
  constructor(size = 4) {
    this.workers = [];
    this.queue = [];
    this.available = [];

    for (let i = 0; i < size; i++) {
      const worker = new Worker('./compress-worker.js');
      this.workers.push(worker);
      this.available.push(worker);
    }
  }

  async compress(data) {
    const worker = await this.getWorker();

    return new Promise((resolve, reject) => {
      worker.once('message', (result) => {
        this.releaseWorker(worker);
        resolve(result);
      });
      worker.once('error', (err) => {
        this.releaseWorker(worker);
        reject(err);
      });
      worker.postMessage(data);
    });
  }

  async getWorker() {
    if (this.available.length > 0) {
      return this.available.pop();
    }

    return new Promise((resolve) => {
      this.queue.push(resolve);
    });
  }

  releaseWorker(worker) {
    if (this.queue.length > 0) {
      const next = this.queue.shift();
      next(worker);
    } else {
      this.available.push(worker);
    }
  }
}

Best Practices#

Compression Choice:
✓ Gzip for general use
✓ Brotli for web content
✓ Deflate for compatibility
✓ Consider content type

Performance:
✓ Use streams for large files
✓ Choose appropriate level
✓ Consider CPU vs size tradeoff
✓ Use worker threads if needed

Error Handling:
✓ Handle corrupted data
✓ Clean up partial files
✓ Validate before decompressing
✓ Set reasonable timeouts

Avoid:
✗ Sync methods for large data
✗ Maximum compression always
✗ Ignoring memory limits
✗ Re-compressing compressed data

The zlib module provides efficient compression using Gzip, Deflate, and Brotli algorithms. Use streams for large files, choose appropriate compression levels, and implement proper error handling. Brotli typically achieves better compression for web content, while Gzip offers wider compatibility.