feat: initial POC with NestJS, hexagonal architecture, vitest tests, docker support

.dockerignore

@@ -0,0 +1,5 @@
+node_modules
+dist
+.git
+*.md
+coverage

CHANGELOG.md

@@ -0,0 +1,16 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+## 0.1.0 (2026-06-18)
+
+Initial POC release:
+
+- NestJS + TypeScript, hexagonal architecture.
+- /v1/health endpoint.
+- /v1/scrape endpoint with:
+  - Direct HTTP fetch adapter.
+  - Browserless fallback for SPAs.
+  - Readability + Turndown Markdown extraction.
+- Vitest test suite (12 tests).
+- Dockerfile and docker-compose.yaml including browserless service.

Dockerfile

@@ -0,0 +1,29 @@
+FROM node:22-alpine AS deps
+WORKDIR /app
+COPY package.json package-lock.json ./
+RUN npm ci --only=production
+
+FROM node:22-alpine AS builder
+WORKDIR /app
+COPY package.json package-lock.json ./
+RUN npm ci
+COPY tsconfig.json nest-cli.json vitest.config.ts ./
+COPY src/ src/
+COPY test/ test/
+RUN npx tsc && npx vitest run --reporter=verbose
+
+FROM node:22-alpine AS runner
+WORKDIR /app
+ENV NODE_ENV=production
+ENV PORT=3000
+
+RUN addgroup -S appgroup && adduser -S appuser -G appgroup
+
+COPY --from=deps /app/node_modules ./node_modules
+COPY --from=builder /app/dist ./dist
+
+USER appuser
+
+EXPOSE 3000
+
+CMD ["node", "dist/src/main.js"]

README.md

@@ -0,0 +1,30 @@
+# Firecrawl Browserless Adapter (POC)
+
+Thin adapter that mimics a subset of the Firecrawl API using:
+- Direct HTTP fetch for SSR/static pages.
+- Browserless (headless Chromium) as fallback / SPA renderer.
+- Readability + Turndown to extract clean Markdown.
+
+Tech stack:
+- NestJS, TypeScript, hexagonal architecture.
+- Vitest for tests.
+- Docker + docker-compose only; no host dependencies required.
+
+Endpoints:
+- GET /v1/health -> { success: true, status: "ok", service: "firecrawl-browserless-adapter" }
+- POST /v1/scrape
+  - Body: { url: string, render?: boolean }
+  - Response: { url, title, markdown, html?, metadata: { source } }
+
+Usage (local):
+- docker compose up --build
+- curl http://localhost:3000/v1/health
+- curl -X POST http://localhost:3000/v1/scrape     -H "Content-Type: application/json"     -d '{"url": "https://example.com"}'
+
+SPA / render mode:
+- Set render=true to force Browserless for SPAs.
+- Without it, the adapter tries direct fetch first and falls back if content looks incomplete.
+
+Notes:
+- This is a POC; not production-hardened.
+- No auth, no rate limiting, basic error handling only.

docker-compose.yaml

@@ -0,0 +1,20 @@
+version: '3.9'
+
+services:
+  firecrawl-adapter:
+    build: .
+    ports:
+      - '3000:3000'
+    environment:
+      - PORT=3000
+      - BROWSERLESS_URL=http://browserless:3000
+    depends_on:
+      browserless:
+        condition: service_started
+
+  browserless:
+    image: ghcr.io/browserless/chromium:v1.65.2-latest
+    ports:
+      - '3001:3000'
+    environment:
+      - TOKEN=firecrawl-poc-token

nest-cli.json

@@ -0,0 +1,9 @@
+{
+  "": "https://json.schemastore.org/nest-cli",
+  "collection": "@nestjs/schematics",
+  "sourceRoot": "src",
+  "compilerOptions": {
+    "deleteOutDir": true,
+    "plugins": ["@nestjs/swagger"]
+  }
+}

package.json

@@ -0,0 +1,35 @@
+{
+  "name": "firecrawl-browserless-adapter",
+  "version": "1.0.0",
+  "description": "Thin Firecrawl-compatible adapter using browserless + direct fetch.",
+  "private": true,
+  "scripts": {
+    "start": "nest start",
+    "start:dev": "nest start --watch",
+    "build": "nest build",
+    "test": "vitest run",
+    "test:cov": "vitest run --coverage"
+  },
+  "dependencies": {
+    "@mozilla/readability": "^0.6.0",
+    "@nestjs/common": "^11.0.1",
+    "@nestjs/core": "^11.0.1",
+    "@nestjs/platform-express": "^11.0.1",
+    "express": "^5.1.0",
+    "jsdom": "^26.1.0",
+    "reflect-metadata": "^0.2.2",
+    "rxjs": "^7.8.2",
+    "turndown": "^7.2.0"
+  },
+  "devDependencies": {
+    "@nestjs/cli": "^11.0.5",
+    "@nestjs/testing": "^11.0.1",
+    "@types/express": "^5.0.1",
+    "@types/jsdom": "^21.0.0",
+    "@types/node": "^22.13.10",
+    "@types/turndown": "^5.0.5",
+    "ts-node": "^10.9.2",
+    "typescript": "^5.8.3",
+    "vitest": "^3.2.4"
+  }
+}

src/app.module.ts

@@ -0,0 +1,18 @@
+import { Module } from '@nestjs/common';
+import { ScrapeUseCase } from './application/use-cases/scrape.use-case';
+import { DirectFetchAdapter } from './infrastructure/adapters/direct-fetch.adapter';
+import { BrowserlessAdapter } from './infrastructure/adapters/browserless.adapter';
+import { ReadabilityMarkdownConverter } from './infrastructure/adapters/readability-markdown.converter';
+import { ScrapeController } from './interfaces/http/scrape.controller';
+
+@Module({
+  imports: [],
+  controllers: [ScrapeController],
+  providers: [
+    ScrapeUseCase,
+    DirectFetchAdapter,
+    BrowserlessAdapter,
+    ReadabilityMarkdownConverter,
+  ],
+})
+export class AppModule {}

src/application/use-cases/scrape.use-case.ts

@@ -0,0 +1,81 @@
+import { Injectable } from '@nestjs/common';
+import type { ScrapeRequest } from '../../domain/models/scrape-request.model';
+import type { ScrapeResponse } from '../../domain/models/scrape-response.model';
+import { DirectFetchAdapter } from '../../infrastructure/adapters/direct-fetch.adapter';
+import { BrowserlessAdapter } from '../../infrastructure/adapters/browserless.adapter';
+import { ReadabilityMarkdownConverter } from '../../infrastructure/adapters/readability-markdown.converter';
+
+@Injectable()
+export class ScrapeUseCase {
+  constructor(
+    private readonly directFetcher: DirectFetchAdapter,
+    private readonly browserlessFetcher: BrowserlessAdapter,
+    private readonly markdownConverter: ReadabilityMarkdownConverter,
+  ) {}
+
+  async execute(req: ScrapeRequest): Promise<ScrapeResponse> {
+    const forceBrowserless = req.render === true;
+    let html: string;
+    let source: 'direct' | 'browserless';
+
+    if (forceBrowserless) {
+      source = 'browserless';
+      html = await this.browserlessFetcher.fetch(req.url);
+    } else {
+      try {
+        html = await this.directFetcher.fetch(req.url);
+
+        if (!this.isContentSufficient(html)) {
+          throw new Error('Direct fetch result looks incomplete');
+        }
+
+        source = 'direct';
+      } catch {
+        source = 'browserless';
+        html = await this.browserlessFetcher.fetch(req.url);
+      }
+    }
+
+    const { title, markdown } = await this.markdownConverter.convert(html, req.url);
+
+    return {
+      url: req.url,
+      title,
+      markdown,
+      html,
+      metadata: { source },
+    };
+  }
+
+  private isContentSufficient(html: string): boolean {
+    if (!html || html.length < 500) {
+      return false;
+    }
+
+    const lower = html.toLowerCase();
+
+    const bodyText = lower
+      .replace(/<script[\s\S]*?<\/script>/g, '')
+      .replace(/<style[\s\S]*?<\/style>/g, '')
+      .replace(/<[^>]+>/g, '')
+      .trim();
+
+    if (bodyText.length < 200) {
+      return false;
+    }
+
+    const spaMarkers = [
+      'id="__next"',
+      'id="root"',
+      'data-reactroot',
+      'window.__nuxt__',
+      '__vite',
+    ];
+
+    if (spaMarkers.some((m) => lower.includes(m)) && bodyText.length < 1000) {
+      return false;
+    }
+
+    return true;
+  }
+}

src/domain/models/scrape-request.model.ts

@@ -0,0 +1,4 @@
+export interface ScrapeRequest {
+  url: string;
+  render?: boolean;
+}

src/domain/models/scrape-response.model.ts

@@ -0,0 +1,9 @@
+export interface ScrapeResponse {
+  url: string;
+  title: string;
+  markdown: string;
+  html?: string;
+  metadata: {
+    source: 'direct' | 'browserless';
+  };
+}

src/domain/ports/html-fetcher.port.ts

@@ -0,0 +1,3 @@
+export interface HtmlFetcher {
+  fetch(url: string): Promise<string>;
+}

src/domain/ports/markdown-converter.port.ts

@@ -0,0 +1,3 @@
+export interface MarkdownConverter {
+  convert(html: string, url: string): Promise<{ title: string; markdown: string }>;
+}

src/domain/ports/spa-detector.port.ts

@@ -0,0 +1,3 @@
+export interface SpaDetector {
+  isLikelySpa(html: string): boolean;
+}

src/infrastructure/adapters/browserless.adapter.ts

@@ -0,0 +1,45 @@
+import { Injectable } from '@nestjs/common';
+import type { HtmlFetcher } from '../../domain/ports/html-fetcher.port';
+
+@Injectable()
+export class BrowserlessAdapter implements HtmlFetcher {
+  private readonly baseUrl: string;
+  private readonly token?: string;
+
+  constructor(
+    baseUrl: string = process.env.BROWSERLESS_URL || 'http://browserless:3000',
+    token: string = process.env.BROWSERLESS_TOKEN || '',
+  ) {
+    this.baseUrl = baseUrl;
+    this.token = token || undefined;
+  }
+
+  private endpoint(path: string): string {
+    const url = new URL(path, this.baseUrl);
+    if (this.token) {
+      url.searchParams.set('token', this.token);
+    }
+    return url.toString();
+  }
+
+  async fetch(targetUrl: string): Promise<string> {
+    const res = await fetch(this.endpoint('/content'), {
+      method: 'POST',
+      headers: { 'content-type': 'application/json' },
+      body: JSON.stringify({
+        url: targetUrl,
+        gotoOptions: {
+          waitUntil: 'domcontentloaded',
+          timeout: 15000,
+        },
+      }),
+    });
+
+    if (!res.ok) {
+      const body = await res.text();
+      throw new Error('Browserless failed: ' + res.status + ' ' + body);
+    }
+
+    return res.text();
+  }
+}

src/infrastructure/adapters/direct-fetch.adapter.ts

@@ -0,0 +1,35 @@
+import { Injectable } from '@nestjs/common';
+import type { HtmlFetcher } from '../../domain/ports/html-fetcher.port';
+
+@Injectable()
+export class DirectFetchAdapter implements HtmlFetcher {
+  async fetch(url: string): Promise<string> {
+    const res = await fetch(url, {
+      headers: {
+        'user-agent':
+          'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/125 Safari/537.36',
+        accept:
+          'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+        'accept-language': 'de-DE,de;q=0.9,en;q=0.8',
+      },
+      redirect: 'follow',
+    });
+
+    if (!res.ok) {
+      throw new Error('Direct fetch failed: ' + res.status);
+    }
+
+    const contentType = res.headers.get('content-type') || '';
+
+    if (
+      !contentType.includes('text/html') &&
+      !contentType.includes('application/xhtml+xml')
+    ) {
+      throw new Error(
+        'Direct fetch returned non-HTML content: ' + contentType,
+      );
+    }
+
+    return res.text();
+  }
+}

src/infrastructure/adapters/readability-markdown.converter.ts

@@ -0,0 +1,33 @@
+import { Injectable } from '@nestjs/common';
+import { JSDOM } from 'jsdom';
+import { Readability } from '@mozilla/readability';
+import TurndownService from 'turndown';
+import type { MarkdownConverter } from '../../domain/ports/markdown-converter.port';
+
+@Injectable()
+export class ReadabilityMarkdownConverter implements MarkdownConverter {
+  async convert(
+    html: string,
+    url: string,
+  ): Promise<{ title: string; markdown: string }> {
+    const dom = new JSDOM(html, { url });
+    const reader = new Readability(dom.window.document);
+    const article = reader.parse();
+
+    const title =
+      article?.title || dom.window.document.title || url;
+    const contentHtml =
+      article?.content ||
+      dom.window.document.body?.innerHTML ||
+      html;
+
+    const turndown = new TurndownService({
+      headingStyle: 'atx',
+      codeBlockStyle: 'fenced',
+    });
+
+    const markdown = turndown.turndown(contentHtml).trim();
+
+    return { title, markdown };
+  }
+}

src/interfaces/http/dto/scrape-request.dto.ts

@@ -0,0 +1,4 @@
+export class ScrapeRequestDto {
+  url?: string;
+  render?: boolean;
+}

src/interfaces/http/scrape.controller.ts

@@ -0,0 +1,31 @@
+import { Controller, Post, Get, Body, HttpCode, HttpStatus } from '@nestjs/common';
+import type { ScrapeResponse } from '../../domain/models/scrape-response.model';
+import { ScrapeUseCase } from '../../application/use-cases/scrape.use-case';
+
+@Controller('v1')
+export class ScrapeController {
+  constructor(private readonly scrapeUseCase: ScrapeUseCase) {}
+
+  @Get('health')
+  health(): { success: boolean; status: string; service: string } {
+    return {
+      success: true,
+      status: 'ok',
+      service: 'firecrawl-browserless-adapter',
+    };
+  }
+
+  @Post('scrape')
+  @HttpCode(HttpStatus.OK)
+  async scrape(@Body() body: any): Promise<ScrapeResponse> {
+    const url = body?.url;
+    if (!url || typeof url !== 'string') {
+      throw new Error('Missing or invalid "url" field');
+    }
+
+    return this.scrapeUseCase.execute({
+      url,
+      render: Boolean(body?.render),
+    });
+  }
+}

src/main.ts

@@ -0,0 +1,11 @@
+import { NestFactory } from '@nestjs/core';
+import { AppModule } from './app.module';
+
+async function bootstrap() {
+  const port = process.env.PORT || '3000';
+  const app = await NestFactory.create(AppModule);
+  await app.listen(port, '0.0.0.0');
+  console.log('Firecrawl adapter listening on ' + port);
+}
+
+bootstrap();

test/browserless.adapter.spec.ts

@@ -0,0 +1,21 @@
+import { BrowserlessAdapter } from '../src//infrastructure/adapters/browserless.adapter';
+
+describe('BrowserlessAdapter', () => {
+  it('uses BROWSERLESS_URL env when provided', () => {
+    const original = process.env.BROWSERLESS_URL;
+    process.env.BROWSERLESS_URL = 'http://custom:9999';
+    const adapter = new BrowserlessAdapter();
+    // we cannot easily inspect private fields, but no crash means it's wired.
+    expect(adapter).toBeDefined();
+    if (original !== undefined) {
+      process.env.BROWSERLESS_URL = original;
+    } else {
+      delete process.env.BROWSERLESS_URL;
+    }
+  });
+
+  it('throws when browserless returns non-ok', async () => {
+    const adapter = new BrowserlessAdapter('http://127.0.0.1:1');
+    await expect(adapter.fetch('https://example.com')).rejects.toThrow();
+  });
+});

test/direct-fetch.adapter.spec.ts

@@ -0,0 +1,24 @@
+import { DirectFetchAdapter } from '../src//infrastructure/adapters/direct-fetch.adapter';
+
+describe('DirectFetchAdapter', () => {
+  let adapter: DirectFetchAdapter;
+
+  beforeEach(() => {
+    adapter = new DirectFetchAdapter();
+  });
+
+  it('fetches HTML from a valid URL', async () => {
+    const html = await adapter.fetch('https://httpbin.org/html');
+    expect(html).toContain('<html>');
+  });
+
+  it('throws on non-200 response', async () => {
+    await expect(adapter.fetch('https://httpbin.org/status/404')).rejects.toThrow();
+  });
+
+  it('throws on non-HTML content type', async () => {
+    await expect(adapter.fetch('https://httpbin.org/json')).rejects.toThrow(
+      'non-HTML'
+    );
+  });
+});

test/readability-markdown.converter.spec.ts

@@ -0,0 +1,30 @@
+import { ReadabilityMarkdownConverter } from '../src//infrastructure/adapters/readability-markdown.converter';
+
+describe('ReadabilityMarkdownConverter', () => {
+  let converter: ReadabilityMarkdownConverter;
+
+  beforeEach(() => {
+    converter = new ReadabilityMarkdownConverter();
+  });
+
+  it('converts HTML to markdown', async () => {
+    const html = '<html><body><h1>Hello</h1><p>World</p></body></html>';
+    const result = await converter.convert(html, 'https://example.com');
+
+    expect(result.markdown).toContain('# Hello');
+  });
+
+  it('extracts title from document', async () => {
+    const html = '<html><head><title>My Title</title></head><body><p>Text</p></body></html>';
+    const result = await converter.convert(html, 'https://example.com');
+
+    expect(result.title).toBe('My Title');
+  });
+
+  it('falls back to URL as title when no title', async () => {
+    const html = '<html><head></head><body><p>Text</p></body></html>';
+    const result = await converter.convert(html, 'https://example.com');
+
+    expect(result.title).toBe('https://example.com');
+  });
+});

test/scrape.use-case.spec.ts

@@ -0,0 +1,76 @@
+import { Test, TestingModule } from '@nestjs/testing';
+import { ScrapeUseCase } from '../src//application/use-cases/scrape.use-case';
+import { DirectFetchAdapter } from '../src//infrastructure/adapters/direct-fetch.adapter';
+import { BrowserlessAdapter } from '../src//infrastructure/adapters/browserless.adapter';
+import { ReadabilityMarkdownConverter } from '../src//infrastructure/adapters/readability-markdown.converter';
+
+describe('ScrapeUseCase', () => {
+  let useCase: ScrapeUseCase;
+  let directMock: Mock<DirectFetchAdapter>;
+  let browserlessMock: Mock<BrowserlessAdapter>;
+  let converterMock: Mock<ReadabilityMarkdownConverter>;
+
+  const sampleHtml = (len: number) => '<html><body>' + 'x'.repeat(len) + '</body></html>';
+
+  beforeEach(async () => {
+    directMock = { fetch: vi.fn() } as any;
+    browserlessMock = { fetch: vi.fn() } as any;
+    converterMock = { convert: vi.fn() } as any;
+
+    const module: TestingModule = await Test.createTestingModule({
+      providers: [
+        ScrapeUseCase,
+        { provide: DirectFetchAdapter, useValue: directMock },
+        { provide: BrowserlessAdapter, useValue: browserlessMock },
+        { provide: ReadabilityMarkdownConverter, useValue: converterMock },
+      ],
+    }).compile();
+
+    useCase = module.get(ScrapeUseCase);
+  });
+
+  it('uses direct fetch when content is sufficient', async () => {
+    const html = sampleHtml(2000);
+    directMock.fetch.mockResolvedValue(html);
+    converterMock.convert.mockResolvedValue({ title: 'T', markdown: 'M' });
+
+    const result = await useCase.execute({ url: 'https://example.com' });
+
+    expect(directMock.fetch).toHaveBeenCalled();
+    expect(browserlessMock.fetch).not.toHaveBeenCalled();
+    expect(result.metadata.source).toBe('direct');
+  });
+
+  it('falls back to browserless when direct content is insufficient', async () => {
+    const shortHtml = '<html><body>short</body></html>';
+    directMock.fetch.mockResolvedValue(shortHtml);
+    browserlessMock.fetch.mockResolvedValue(sampleHtml(2000));
+    converterMock.convert.mockResolvedValue({ title: 'T', markdown: 'M' });
+
+    const result = await useCase.execute({ url: 'https://example.com' });
+
+    expect(browserlessMock.fetch).toHaveBeenCalled();
+    expect(result.metadata.source).toBe('browserless');
+  });
+
+  it('uses browserless directly when render=true', async () => {
+    browserlessMock.fetch.mockResolvedValue(sampleHtml(2000));
+    converterMock.convert.mockResolvedValue({ title: 'T', markdown: 'M' });
+
+    const result = await useCase.execute({ url: 'https://example.com', render: true });
+
+    expect(directMock.fetch).not.toHaveBeenCalled();
+    expect(browserlessMock.fetch).toHaveBeenCalled();
+    expect(result.metadata.source).toBe('browserless');
+  });
+
+  it('returns markdown and title from converter', async () => {
+    directMock.fetch.mockResolvedValue(sampleHtml(2000));
+    converterMock.convert.mockResolvedValue({ title: 'Page Title', markdown: '# Hello' });
+
+    const result = await useCase.execute({ url: 'https://example.com' });
+
+    expect(result.title).toBe('Page Title');
+    expect(result.markdown).toBe('# Hello');
+  });
+});

tsconfig.json

@@ -0,0 +1,28 @@
+{
+  "compilerOptions": {
+    "module": "CommonJS",
+    "moduleResolution": "Node",
+    "target": "ES2022",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "declaration": true,
+    "resolveJsonModule": true,
+    "emitDecoratorMetadata": true,
+    "experimentalDecorators": true,
+    "types": [
+      "node"
+    ]
+  },
+  "include": [
+    "src/**/*"
+  ],
+  "exclude": [
+    "node_modules",
+    "dist",
+    "test"
+  ]
+}

vitest.config.ts

@@ -0,0 +1,14 @@
+import { defineConfig } from 'vitest/config';
+
+export default defineConfig({
+  test: {
+    globals: true,
+    environment: 'node',
+    include: ['test/**/*.spec.ts'],
+    coverage: {
+      provider: 'v8',
+      include: ['src/**/*.{ts,js}'],
+      exclude: ['src/main.ts', 'src/**/*.module.ts'],
+    },
+  },
+});