Node.js Child Processes

Child processes allow Node.js to run external commands and parallelize work. Here's how to use them effectively.

Overview of Methods#

const { exec, execFile, spawn, fork } = require('child_process');

// exec: Run command in shell, buffer output
// execFile: Run executable directly, buffer output
// spawn: Run command, stream output
// fork: Special spawn for Node.js scripts with IPC

Using exec#

const { exec } = require('child_process');

// Basic usage
exec('ls -la', (error, stdout, stderr) => {
  if (error) {
    console.error(`Error: ${error.message}`);
    return;
  }
  if (stderr) {
    console.error(`stderr: ${stderr}`);
    return;
  }
  console.log(`stdout: ${stdout}`);
});

// With options
exec('cat package.json', {
  cwd: '/path/to/project',
  encoding: 'utf8',
  maxBuffer: 1024 * 1024, // 1MB
  timeout: 5000,
  env: { ...process.env, NODE_ENV: 'production' },
}, (error, stdout, stderr) => {
  console.log(stdout);
});

// Promise wrapper
const { promisify } = require('util');
const execAsync = promisify(exec);

async function runCommand(cmd) {
  try {
    const { stdout, stderr } = await execAsync(cmd);
    return stdout.trim();
  } catch (error) {
    throw new Error(`Command failed: ${error.message}`);
  }
}

// Usage
const gitBranch = await runCommand('git branch --show-current');
console.log(`Current branch: ${gitBranch}`);

Using execFile#

const { execFile } = require('child_process');

// More secure - doesn't use shell
execFile('node', ['--version'], (error, stdout, stderr) => {
  if (error) {
    console.error(error);
    return;
  }
  console.log(`Node version: ${stdout}`);
});

// Run a script
execFile('./script.sh', ['arg1', 'arg2'], {
  cwd: __dirname,
}, (error, stdout, stderr) => {
  console.log(stdout);
});

// Promise version
const { promisify } = require('util');
const execFileAsync = promisify(execFile);

async function runScript() {
  const { stdout } = await execFileAsync('python3', ['script.py', '--flag']);
  return JSON.parse(stdout);
}

Using spawn#

const { spawn } = require('child_process');

// Basic spawn
const ls = spawn('ls', ['-la']);

ls.stdout.on('data', (data) => {
  console.log(`stdout: ${data}`);
});

ls.stderr.on('data', (data) => {
  console.error(`stderr: ${data}`);
});

ls.on('close', (code) => {
  console.log(`Process exited with code ${code}`);
});

// With options
const child = spawn('npm', ['install'], {
  cwd: '/path/to/project',
  env: { ...process.env, CI: 'true' },
  stdio: 'inherit', // Inherit parent's stdio
});

// Pipe streams
const grep = spawn('grep', ['pattern']);
const cat = spawn('cat', ['file.txt']);

cat.stdout.pipe(grep.stdin);

grep.stdout.on('data', (data) => {
  console.log(`Found: ${data}`);
});

// Detached process
const child = spawn('long-running-process', [], {
  detached: true,
  stdio: 'ignore',
});

child.unref(); // Allow parent to exit

// Shell command
const child = spawn('echo "Hello" && echo "World"', {
  shell: true,
});

Using fork#

// parent.js
const { fork } = require('child_process');

const child = fork('worker.js');

// Send message to child
child.send({ type: 'START', data: [1, 2, 3, 4, 5] });

// Receive messages from child
child.on('message', (message) => {
  console.log('Result from child:', message);
});

child.on('exit', (code) => {
  console.log(`Child exited with code ${code}`);
});

// worker.js
process.on('message', (message) => {
  if (message.type === 'START') {
    const result = message.data.reduce((a, b) => a + b, 0);
    process.send({ type: 'RESULT', data: result });
  }
});

// With module path
const worker = fork('./worker.js', ['--arg1'], {
  cwd: __dirname,
  env: { ...process.env, WORKER_ID: '1' },
  execArgv: ['--max-old-space-size=4096'],
});

Process Pool#

const { fork } = require('child_process');
const os = require('os');

class ProcessPool {
  constructor(workerPath, poolSize = os.cpus().length) {
    this.workerPath = workerPath;
    this.poolSize = poolSize;
    this.workers = [];
    this.taskQueue = [];
    this.activeWorkers = new Map();

    this.initialize();
  }

  initialize() {
    for (let i = 0; i < this.poolSize; i++) {
      this.createWorker();
    }
  }

  createWorker() {
    const worker = fork(this.workerPath);

    worker.on('message', (result) => {
      const { resolve, reject } = this.activeWorkers.get(worker);

      if (result.error) {
        reject(new Error(result.error));
      } else {
        resolve(result.data);
      }

      this.activeWorkers.delete(worker);
      this.workers.push(worker);
      this.processQueue();
    });

    worker.on('error', (error) => {
      const task = this.activeWorkers.get(worker);
      if (task) {
        task.reject(error);
        this.activeWorkers.delete(worker);
      }
      this.createWorker(); // Replace dead worker
    });

    this.workers.push(worker);
  }

  async execute(data) {
    return new Promise((resolve, reject) => {
      this.taskQueue.push({ data, resolve, reject });
      this.processQueue();
    });
  }

  processQueue() {
    while (this.workers.length > 0 && this.taskQueue.length > 0) {
      const worker = this.workers.pop();
      const task = this.taskQueue.shift();

      this.activeWorkers.set(worker, task);
      worker.send(task.data);
    }
  }

  shutdown() {
    for (const worker of this.workers) {
      worker.kill();
    }
    for (const worker of this.activeWorkers.keys()) {
      worker.kill();
    }
  }
}

// worker.js
process.on('message', async (data) => {
  try {
    const result = await processData(data);
    process.send({ data: result });
  } catch (error) {
    process.send({ error: error.message });
  }
});

// Usage
const pool = new ProcessPool('./worker.js', 4);

const results = await Promise.all([
  pool.execute({ task: 'compute', value: 100 }),
  pool.execute({ task: 'compute', value: 200 }),
  pool.execute({ task: 'compute', value: 300 }),
]);

Streaming Large Data#

const { spawn } = require('child_process');
const fs = require('fs');

// Process large file with external tool
function processLargeFile(inputPath, outputPath) {
  return new Promise((resolve, reject) => {
    const input = fs.createReadStream(inputPath);
    const output = fs.createWriteStream(outputPath);

    const gzip = spawn('gzip', ['-c']);

    input.pipe(gzip.stdin);
    gzip.stdout.pipe(output);

    gzip.on('close', (code) => {
      if (code === 0) {
        resolve();
      } else {
        reject(new Error(`gzip exited with code ${code}`));
      }
    });

    gzip.on('error', reject);
  });
}

// FFmpeg example
function convertVideo(input, output) {
  return new Promise((resolve, reject) => {
    const ffmpeg = spawn('ffmpeg', [
      '-i', input,
      '-c:v', 'libx264',
      '-preset', 'fast',
      '-c:a', 'aac',
      output,
    ]);

    ffmpeg.stderr.on('data', (data) => {
      // FFmpeg outputs progress to stderr
      const progress = parseProgress(data.toString());
      if (progress) {
        console.log(`Progress: ${progress}%`);
      }
    });

    ffmpeg.on('close', (code) => {
      code === 0 ? resolve() : reject(new Error('Conversion failed'));
    });
  });
}

Error Handling#

const { spawn, exec } = require('child_process');

// Comprehensive error handling
function runCommand(command, args = []) {
  return new Promise((resolve, reject) => {
    const child = spawn(command, args);

    let stdout = '';
    let stderr = '';

    child.stdout.on('data', (data) => {
      stdout += data;
    });

    child.stderr.on('data', (data) => {
      stderr += data;
    });

    child.on('error', (error) => {
      reject(new Error(`Failed to start: ${error.message}`));
    });

    child.on('close', (code, signal) => {
      if (signal) {
        reject(new Error(`Process killed with signal ${signal}`));
      } else if (code !== 0) {
        const error = new Error(`Process exited with code ${code}`);
        error.stdout = stdout;
        error.stderr = stderr;
        reject(error);
      } else {
        resolve({ stdout, stderr });
      }
    });
  });
}

// Timeout handling
function runWithTimeout(command, timeout = 30000) {
  return new Promise((resolve, reject) => {
    const child = exec(command, { timeout });

    child.on('error', reject);

    child.on('close', (code, signal) => {
      if (signal === 'SIGTERM') {
        reject(new Error('Process timed out'));
      } else if (code === 0) {
        resolve();
      } else {
        reject(new Error(`Exit code: ${code}`));
      }
    });
  });
}

IPC Communication#

// parent.js
const { fork } = require('child_process');

const child = fork('worker.js', [], {
  serialization: 'advanced', // Support more data types
});

// Send various data types
child.send({
  buffer: Buffer.from('hello'),
  date: new Date(),
  map: new Map([['key', 'value']]),
});

// Bidirectional communication
child.on('message', (msg) => {
  if (msg.type === 'REQUEST_DATA') {
    child.send({ type: 'DATA', payload: getData() });
  }
});

// Send file descriptor
const net = require('net');
const server = net.createServer();

server.listen(0, () => {
  child.send('server', server); // Pass server handle
});

// worker.js
process.on('message', (msg, handle) => {
  if (handle) {
    // Received a socket or server
    handle.on('connection', (socket) => {
      // Handle connection in child process
    });
  }
});

Cluster Module Integration#

const cluster = require('cluster');
const http = require('http');
const os = require('os');

if (cluster.isPrimary) {
  const numCPUs = os.cpus().length;

  console.log(`Primary ${process.pid} is running`);

  // Fork workers
  for (let i = 0; i < numCPUs; i++) {
    cluster.fork();
  }

  cluster.on('exit', (worker, code, signal) => {
    console.log(`Worker ${worker.process.pid} died`);
    cluster.fork(); // Replace dead worker
  });

  // Message from workers
  cluster.on('message', (worker, message) => {
    console.log(`Message from worker ${worker.id}:`, message);
  });

} else {
  // Workers share TCP connection
  http.createServer((req, res) => {
    res.writeHead(200);
    res.end(`Hello from worker ${cluster.worker.id}\n`);
  }).listen(8000);

  console.log(`Worker ${process.pid} started`);

  // Send message to primary
  process.send({ type: 'ready', pid: process.pid });
}

Best Practices#

Method Selection:
✓ exec: Simple commands, need shell features
✓ execFile: Known executables, security matters
✓ spawn: Large output, streaming needed
✓ fork: Node.js scripts with IPC

Security:
✓ Prefer execFile over exec
✓ Validate user input
✓ Use shell: false when possible
✓ Set appropriate cwd and env

Performance:
✓ Use process pools for heavy work
✓ Stream large data instead of buffering
✓ Limit concurrent processes
✓ Handle cleanup properly

Error Handling:
✓ Listen to 'error' events
✓ Check exit codes and signals
✓ Handle timeouts
✓ Log stderr appropriately

Child processes enable Node.js to run external commands and parallelize CPU-intensive work. Use exec for simple shell commands, spawn for streaming, and fork for Node.js worker processes with IPC. Implement process pools for heavy workloads and always handle errors appropriately.