Basic Usage

A single end-to-end example showing the two core operations — scrape a page, then crawl a small site — in every supported binding. Each tab is the canonical “hello world” for that language: create_engine, scrape(engine, url), CrawlConfig(max_depth=1, max_pages=5), crawl(engine, url).

use crawlberg::{crawl, create_engine, scrape, CrawlConfig, CrawlError};

#[tokio::main]
async fn main() -> Result<(), CrawlError> {
    // Simplest case: scrape a single page with default settings.
    let engine = create_engine(None)?;
    let result = scrape(&engine, "https://example.com/").await?;
    println!("Title: {}", result.metadata.title.as_deref().unwrap_or(""));
    println!("Status: {}", result.status_code);
    println!("Links found: {}", result.links.len());

    // Crawl from a seed URL, limited to one hop and a handful of pages.
    let config = CrawlConfig::builder().max_depth(1).max_pages(5).build();
    let crawl_engine = create_engine(Some(config))?;
    let crawl_result = crawl(&crawl_engine, "https://en.wikipedia.org/wiki/Web_scraping").await?;
    println!("Pages crawled: {}", crawl_result.pages.len());

    Ok(())
}

import asyncio

from crawlberg import CrawlConfig, create_engine, crawl, scrape


async def main() -> None:
    # Simplest case: scrape a single page with default settings.
    engine = create_engine()
    result = await scrape(engine, "https://example.com/")
    print(f"Title: {result.metadata.title}")
    print(f"Status: {result.status_code}")
    print(f"Links found: {len(result.links)}")

    # Crawl from a seed URL, limited to one hop and a handful of pages.
    crawl_engine = create_engine(CrawlConfig(max_depth=1, max_pages=5))
    crawl_result = await crawl(crawl_engine, "https://en.wikipedia.org/wiki/Web_scraping")
    print(f"Pages crawled: {len(crawl_result.pages)}")


if __name__ == "__main__":
    asyncio.run(main())

import { crawl, createEngine, scrape } from "@xberg-io/crawlberg";

async function main(): Promise<void> {
  // Simplest case: scrape a single page with default settings.
  const engine = createEngine();
  const result = await scrape(engine, "https://example.com/");
  console.log(`Title: ${result.metadata?.title ?? ""}`);
  console.log(`Status: ${result.statusCode}`);
  console.log(`Links found: ${result.links?.length ?? 0}`);

  // Crawl from a seed URL, limited to one hop and a handful of pages.
  const crawlEngine = createEngine({ maxDepth: 1, maxPages: 5 });
  const crawlResult = await crawl(crawlEngine, "https://en.wikipedia.org/wiki/Web_scraping");
  console.log(`Pages crawled: ${crawlResult.pages?.length ?? 0}`);
}

main().catch((error) => {
  console.error(error);
  process.exit(1);
});

require "crawlberg"

# Simplest case: scrape a single page with default settings.
engine = Crawlberg.create_engine
result = Crawlberg.scrape(engine, "https://example.com/")
puts "Title: #{result.metadata.title}"
puts "Status: #{result.status_code}"
puts "Links found: #{result.links.length}"

# Crawl from a seed URL, limited to one hop and a handful of pages.
config = Crawlberg::CrawlConfig.new(max_depth: 1, max_pages: 5)
crawl_engine = Crawlberg.create_engine(config)
crawl_result = Crawlberg.crawl(crawl_engine, "https://en.wikipedia.org/wiki/Web_scraping")
puts "Pages crawled: #{crawl_result.pages.length}"

package main

import (
    "fmt"
    "log"

    crawlberg "github.com/xberg-io/crawlberg/packages/go"
)

func main() {
    // Simplest case: scrape a single page with default settings.
    engine, err := crawlberg.CreateEngine(nil)
    if err != nil {
        log.Fatalf("create engine: %v", err)
    }

    result, err := crawlberg.Scrape(engine, "https://example.com/")
    if err != nil {
        log.Fatalf("scrape: %v", err)
    }
    title := ""
    if result.Metadata.Title != nil {
        title = *result.Metadata.Title
    }
    fmt.Printf("Title: %s\n", title)
    fmt.Printf("Status: %d\n", result.StatusCode)
    fmt.Printf("Links found: %d\n", len(result.Links))

    // Crawl from a seed URL, limited to one hop and a handful of pages.
    config := &crawlberg.CrawlConfig{
        MaxDepth: crawlberg.Ptr(uint(1)),
        MaxPages: crawlberg.Ptr(uint(5)),
    }
    crawlEngine, err := crawlberg.CreateEngine(config)
    if err != nil {
        log.Fatalf("create crawl engine: %v", err)
    }
    crawlResult, err := crawlberg.Crawl(crawlEngine, "https://en.wikipedia.org/wiki/Web_scraping")
    if err != nil {
        log.Fatalf("crawl: %v", err)
    }
    fmt.Printf("Pages crawled: %d\n", len(crawlResult.Pages))
}

import io.xberg.crawlberg.CrawlConfig;
import io.xberg.crawlberg.CrawlEngineHandle;
import io.xberg.crawlberg.CrawlResult;
import io.xberg.crawlberg.Crawlberg;
import io.xberg.crawlberg.ScrapeResult;

public final class BasicUsage {
    private BasicUsage() { }

    public static void main(final String[] args) throws Exception {
        // Simplest case: scrape a single page with default settings.
        CrawlEngineHandle engine = Crawlberg.createEngine();
        ScrapeResult result = Crawlberg.scrape(engine, "https://example.com/");
        System.out.println("Title: " + result.metadata().title());
        System.out.println("Status: " + result.statusCode());
        System.out.println("Links found: " + result.links().size());

        // Crawl from a seed URL, limited to one hop and a handful of pages.
        CrawlConfig config = CrawlConfig.builder()
            .withMaxDepth(1L)
            .withMaxPages(5L)
            .build();
        CrawlEngineHandle crawlEngine = Crawlberg.createEngine(config);
        CrawlResult crawlResult = Crawlberg.crawl(
            crawlEngine,
            "https://en.wikipedia.org/wiki/Web_scraping"
        );
        System.out.println("Pages crawled: " + crawlResult.pages().size());
    }
}

using System;
using System.Threading.Tasks;

using Crawlberg;

internal static class BasicUsage
{
    public static async Task Main()
    {
        // Simplest case: scrape a single page with default settings.
        var engine = CrawlbergConverter.CreateEngine(null);
        var result = await CrawlbergConverter.ScrapeAsync(engine, "https://example.com/");
        Console.WriteLine($"Title: {result.Metadata.Title}");
        Console.WriteLine($"Status: {result.StatusCode}");
        Console.WriteLine($"Links found: {result.Links.Count}");

        // Crawl from a seed URL, limited to one hop and a handful of pages.
        var config = new CrawlConfig
        {
            MaxDepth = 1,
            MaxPages = 5,
        };
        var crawlEngine = CrawlbergConverter.CreateEngine(config);
        var crawlResult = await CrawlbergConverter.CrawlAsync(
            crawlEngine,
            "https://en.wikipedia.org/wiki/Web_scraping"
        );
        Console.WriteLine($"Pages crawled: {crawlResult.Pages.Count}");
    }
}

<?php
declare(strict_types=1);

use Crawlberg\CrawlConfig;
use Crawlberg\Crawlberg;

// Simplest case: scrape a single page with default settings.
$engine = Crawlberg::createEngine(null);
$result = Crawlberg::scrape($engine, "https://example.com/");
echo "Title: " . ($result->getMetadata()->title ?? "") . "\n";
echo "Status: " . $result->statusCode . "\n";
echo "Links found: " . count($result->getLinks()) . "\n";

// Crawl from a seed URL, limited to one hop and a handful of pages.
$config = CrawlConfig::default();
$config->maxDepth = 1;
$config->maxPages = 5;
$crawlEngine = Crawlberg::createEngine($config);
$crawlResult = Crawlberg::crawl($crawlEngine, "https://en.wikipedia.org/wiki/Web_scraping");
echo "Pages crawled: " . count($crawlResult->getPages()) . "\n";

# Simplest case: scrape a single page with default settings.
{:ok, engine} = Crawlberg.create_engine()
{:ok, scrape} = Crawlberg.scrape(engine, "https://example.com/")
IO.puts("Title: #{scrape.metadata.title}")
IO.puts("Status: #{scrape.status_code}")
IO.puts("Links found: #{length(scrape.links)}")

# Crawl from a seed URL, limited to one hop and a handful of pages.
config_json = Jason.encode!(%Crawlberg.CrawlConfig{max_depth: 1, max_pages: 5})
{:ok, crawl_engine} = Crawlberg.create_engine(config_json)
{:ok, crawl} = Crawlberg.crawl(crawl_engine, "https://en.wikipedia.org/wiki/Web_scraping")
IO.puts("Pages crawled: #{length(crawl.pages)}")

import { WasmCrawlConfig, crawl, createEngine, scrape } from "@xberg-io/crawlberg-wasm";

async function main() {
  // Simplest case: scrape a single page with default settings.
  const engine = createEngine();
  const result = await scrape(engine, "https://example.com/");
  console.log(`Title: ${result.metadata?.title ?? ""}`);
  console.log(`Status: ${result.statusCode}`);
  console.log(`Links found: ${result.links?.length ?? 0}`);

  // Crawl from a seed URL, limited to one hop and a handful of pages.
  const config = WasmCrawlConfig.default();
  config.maxDepth = 1;
  config.maxPages = 5;
  const crawlEngine = createEngine(config);
  const crawlResult = await crawl(crawlEngine, "https://en.wikipedia.org/wiki/Web_scraping");
  console.log(`Pages crawled: ${crawlResult.pages?.length ?? 0}`);
}

main().catch((error) => console.error(error));

import 'package:crawlberg/crawlberg.dart';
import 'package:crawlberg/src/crawlberg_bridge_generated/frb_generated.dart'
    show RustLib;

Future<void> main() async {
  await RustLib.init();

  // Simplest case: scrape a single page with default settings.
  final engine = await CrawlbergBridge.createEngine();
  final result = await CrawlbergBridge.scrape(engine, 'https://example.com/');
  print('Title: ${result.metadata.title ?? ''}');
  print('Status: ${result.statusCode}');
  print('Links found: ${result.links.length}');

  // Crawl from a seed URL, limited to one hop and a handful of pages.
  final crawlConfig = await createCrawlConfigFromJson(
    json: r'{"max_depth":1,"max_pages":5}',
  );
  final crawlEngine = await CrawlbergBridge.createEngine(config: crawlConfig);
  final crawlResult = await CrawlbergBridge.crawl(
    crawlEngine,
    'https://en.wikipedia.org/wiki/Web_scraping',
  );
  print('Pages crawled: ${crawlResult.pages.length}');
}

import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.databind.PropertyNamingStrategies
import com.fasterxml.jackson.module.kotlin.registerKotlinModule
import io.xberg.crawlberg.android.CrawlConfig
import io.xberg.crawlberg.android.Crawlberg
import kotlinx.coroutines.runBlocking

fun main() = runBlocking {
    // Simplest case: scrape a single page with default settings.
    val engine = Crawlberg.createEngine()
    val result = Crawlberg.scrapeAsync(engine, "https://example.com/")
    println("Title: ${result.metadata.title}")
    println("Status: ${result.statusCode}")
    println("Links found: ${result.links.size}")

    // Crawl from a seed URL, limited to one hop and a handful of pages.
    val mapper = ObjectMapper()
        .registerKotlinModule()
        .setPropertyNamingStrategy(PropertyNamingStrategies.SNAKE_CASE)
    val config = mapper.readValue(
        "{\"max_depth\":1,\"max_pages\":5}",
        CrawlConfig::class.java,
    )
    val crawlEngine = Crawlberg.createEngine(config)
    val crawlResult = Crawlberg.crawlAsync(
        crawlEngine,
        "https://en.wikipedia.org/wiki/Web_scraping",
    )
    println("Pages crawled: ${crawlResult.pages.size}")
}

import Foundation
import Crawlberg

@main
struct BasicUsage {
    static func main() async throws {
        // Simplest case: scrape a single page with default settings.
        let engine = try createEngine()
        let result = try await scrape(engine: engine, url: "https://example.com/")
        print("Title: \(result.metadata().title()?.toString() ?? "")")
        print("Status: \(result.statusCode())")
        print("Links found: \(result.links().count)")

        // Crawl from a seed URL, limited to one hop and a handful of pages.
        let crawlConfig = try crawlConfigFromJson("{\"max_depth\":1,\"max_pages\":5}")
        let crawlEngine = try createEngine(config: crawlConfig)
        let crawlResult = try await crawl(engine: crawlEngine, url: "https://en.wikipedia.org/wiki/Web_scraping")
        print("Pages crawled: \(crawlResult.pages().count)")
    }
}

const std = @import("std");
const crawlberg = @import("crawlberg");

pub fn main() !void {
    var gpa: std.heap.DebugAllocator(.{}) = .init;
    defer _ = gpa.deinit();
    const allocator = gpa.allocator();

    // Simplest case: scrape a single page with default settings.
    const scrape_json = try crawlberg.scrape(null, "https://example.com/");
    defer std.heap.c_allocator.free(scrape_json);
    var scrape_parsed = try std.json.parseFromSlice(std.json.Value, allocator, scrape_json, .{});
    defer scrape_parsed.deinit();
    const result = &scrape_parsed.value;
    const title = result.object.get("metadata").?.object.get("title").?;
    std.debug.print("Title: {s}\n", .{if (title == .string) title.string else ""});
    std.debug.print("Status: {d}\n", .{result.object.get("status_code").?.integer});
    std.debug.print("Links found: {d}\n", .{result.object.get("links").?.array.items.len});

    // Crawl from a seed URL, limited to one hop and a handful of pages.
    const crawl_json = try crawlberg.crawl(
        "{\"max_depth\":1,\"max_pages\":5}",
        "https://en.wikipedia.org/wiki/Web_scraping",
    );
    defer std.heap.c_allocator.free(crawl_json);
    var crawl_parsed = try std.json.parseFromSlice(std.json.Value, allocator, crawl_json, .{});
    defer crawl_parsed.deinit();
    const crawl_result = &crawl_parsed.value;
    std.debug.print("Pages crawled: {d}\n", .{crawl_result.object.get("pages").?.array.items.len});
}

For deeper walkthroughs of scrape, crawl, and map operations — including configuration options, link filtering, and result-shape details — see the Quick Start and the per-operation guides under Guides → Core.