diff --git a/.github/linkchecker/linkchecker.conf b/.github/linkchecker/linkchecker.conf deleted file mode 100644 index 1a2c716..0000000 --- a/.github/linkchecker/linkchecker.conf +++ /dev/null @@ -1,15 +0,0 @@ -[output] -log=html - -[checking] -sslverify=0 - -[filtering] -checkextern=1 -ignorewarnings=http-redirected -ignore= - https://harlequin\.sh.* - https://.*medium\.com.* - https://ibis-project\.org.* - https://enrich\.sh.* - https://streams\.dbconvert\.com.* diff --git a/.github/lychee.toml b/.github/lychee.toml new file mode 100644 index 0000000..e67312b --- /dev/null +++ b/.github/lychee.toml @@ -0,0 +1,16 @@ +no_progress = true +max_retries = 3 +retry_wait_time = 2 + +# These sites reject automated link checks even though the pages are available. +exclude = [ + '^https://harlequin\.sh', + '^https://.*medium\.com', + '^https://ibis-project\.org', + '^https://lib\.rs', + '^https://enrich\.sh', + '^https://streams\.dbconvert\.com', + '^https://communities\.sas\.com', + '^https://www\.packtpub\.com', + '^https://www\.transfermarkt\.co\.uk', +] diff --git a/.github/scripts/check-readme-order.mjs b/.github/scripts/check-readme-order.mjs new file mode 100644 index 0000000..6a5b0e6 --- /dev/null +++ b/.github/scripts/check-readme-order.mjs @@ -0,0 +1,96 @@ +#!/usr/bin/env node + +import { readFileSync } from "node:fs"; + +const [basePath, currentPath] = process.argv.slice(2); +if (!basePath || !currentPath) { + console.error("Usage: check-readme-order.mjs BASE_README CURRENT_README"); + process.exit(2); +} + +const SORTED_SECTION = "Community Extensions"; +const IGNORED_SECTIONS = new Set(["Contents"]); + +function headingName(markdown) { + const linkedHeading = markdown.match(/^\[([^\]]+)\]\([^)]+\)$/); + return linkedHeading ? linkedHeading[1] : markdown; +} + +function parseSections(path) { + const sections = new Map(); + let section = null; + + for (const line of readFileSync(path, "utf8").split("\n")) { + const heading = line.match(/^#{2,3}\s+(.+?)\s*$/); + if (heading) { + section = headingName(heading[1]); + if (!sections.has(section)) { + sections.set(section, []); + } + continue; + } + + if (!section || !line.startsWith("- [")) { + continue; + } + + const item = line.match(/^- \[(.*?)\]\(([^)\s]+)(?:\s+[^)]*)?\)/); + if (!item) { + continue; + } + + const label = item[1].replaceAll("`", "").trim(); + sections.get(section).push({ + sortKey: label.toLowerCase(), + url: item[2], + }); + } + + return sections; +} + +const baseSections = parseSections(basePath); +const currentSections = parseSections(currentPath); +let orderingIsValid = true; + +const sortedEntries = currentSections.get(SORTED_SECTION) ?? []; +for (let index = 1; index < sortedEntries.length; index += 1) { + if (sortedEntries[index - 1].sortKey > sortedEntries[index].sortKey) { + orderingIsValid = false; + } +} + +for (const [section, currentEntries] of currentSections) { + if (section === SORTED_SECTION || IGNORED_SECTIONS.has(section)) { + continue; + } + + const baseEntries = baseSections.get(section) ?? []; + if (baseEntries.length === 0) { + continue; + } + + const isExisting = (entry) => + baseEntries.some( + (baseEntry) => + baseEntry.url === entry.url || baseEntry.sortKey === entry.sortKey, + ); + + let foundNewEntry = false; + for (const entry of currentEntries) { + const entryAlreadyExists = isExisting(entry); + if (entryAlreadyExists && foundNewEntry) { + orderingIsValid = false; + } else if (!entryAlreadyExists) { + foundNewEntry = true; + } + } +} + +if (!orderingIsValid) { + const message = "Thanks for contributing! Please move your new entry to follow the ordering rules in CONTRIBUTING.md, then update this pull request."; + console.error(`::error file=${currentPath},title=README entry order::${message}`); + process.exit(1); +} + +console.log("README entries follow the required ordering."); diff --git a/.github/workflows/linkchecker.yml b/.github/workflows/linkchecker.yml index c15ef2b..b028c1b 100644 --- a/.github/workflows/linkchecker.yml +++ b/.github/workflows/linkchecker.yml @@ -1,31 +1,46 @@ -name: Check links in the README +name: Check the README -on: [push, pull_request] +on: + push: + branches: [main] + paths: + - README.md + - .github/lychee.toml + - .github/scripts/check-readme-order.mjs + - .github/workflows/linkchecker.yml + pull_request: + paths: + - README.md + - .github/lychee.toml + - .github/scripts/check-readme-order.mjs + - .github/workflows/linkchecker.yml + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: link-check-${{ github.ref }} + cancel-in-progress: true jobs: LinkChecker: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest + timeout-minutes: 5 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Check entry order + if: github.event_name == 'pull_request' + env: + BASE_REF: ${{ github.base_ref }} + run: | + git fetch --no-tags --depth=1 origin "$BASE_REF" + git show FETCH_HEAD:README.md > "$RUNNER_TEMP/README.base.md" + node .github/scripts/check-readme-order.mjs "$RUNNER_TEMP/README.base.md" README.md + + - name: Check links + uses: lycheeverse/lychee-action@e7477775783ea5526144ba13e8db5eec57747ce8 # v2.9.0 with: - fetch-depth: 0 - - - uses: awalsh128/cache-apt-pkgs-action@latest - with: - packages: pandoc - - - name: Convert Markdown to HTML format - run: pandoc README.md --standalone --output README.html - - - name: Set up Python 3.9 - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - - run: pip install linkchecker --user - - - name: Link Checker - run: linkchecker README.html --config .github/linkchecker/linkchecker.conf --file-output html/utf-8/report.html - - - run: cat report.html >> ${GITHUB_STEP_SUMMARY} - if: success() || failure() + args: --config .github/lychee.toml README.md + lycheeVersion: v0.24.2 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c7f38e6..2a05032 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,25 +1,28 @@ # Contribution Guidelines -Please note that this project is released with a -[Contributor Code of Conduct](CODE_OF_CONDUCT.md). By participating in this -project you agree to abide by its terms. +By participating in this project, you agree to abide by its [Contributor Code of Conduct](CODE_OF_CONDUCT.md). ---- +## Adding an entry -Ensure your pull request adheres to the following guidelines: +- Check that the project or resource is not already listed. +- Choose the most specific existing section. +- Use the canonical project or repository URL. +- Follow the surrounding Markdown format. Most entries use: + `- [Name](https://example.com) - A concise, factual description.` +- Describe what the project does and how it relates to DuckDB. Avoid promotional claims. +- Keep the pull request focused and disclose your affiliation when submitting your own project. -- Make sure you take care of this -- And this as well -- And don't forget to check this +### Entry order -Thank you for your suggestions! +Most sections preserve insertion order. **Append new entries to the bottom of the relevant list**, even when another entry seems similar. Do not alphabetize or otherwise reposition existing entries. +The **Community Extensions** section is the exception: insert extensions in case-insensitive lexicographic order by extension name. Community extension names use backticks, for example: +``- [`extension_name`](https://example.com) - A concise description.`` -## Updating your PR +When updating an existing entry, leave it in its current position. When moving an entry to a different section, place it according to the destination section's rule. -A lot of times, making a PR adhere to the standards above can be difficult. -If the maintainers notice anything that we'd like changed, we'll ask you to -edit your PR before we merge it. There's no need to open a new PR, just edit -the existing one. If you're not sure how to do that, -[here is a guide](https://github.com/RichardLitt/knowledge/blob/master/github/amending-a-commit-guide.md) -on the different ways you can update your PR so that we can merge it. +## Before opening a pull request + +- Check the Markdown diff for unrelated changes. +- Verify that every new URL works without authentication. +- Confirm that each new entry is appended, or lexicographically placed when it is a Community Extension. diff --git a/README.md b/README.md index 1e0def8..7776d21 100644 --- a/README.md +++ b/README.md @@ -337,8 +337,8 @@ Official DuckDB extensions, which can installed via `INSTALL ⟨extension_name Community-contributed DuckDB extensions, which can be installed via `INSTALL ⟨extension_name⟩ FROM community`. -- [`acp`](https://github.com/sidequery/duckdb-acp) - Embeds AI agents such as Claude Code inside of DuckDB via Agent Client Protocol. - [`3fs`](https://github.com/open3fs/duckdb-3fs) - Integrates DuckDB with DeepSeek 3FS distributed file system. +- [`acp`](https://github.com/sidequery/duckdb-acp) - Embeds AI agents such as Claude Code inside of DuckDB via Agent Client Protocol. - [`bigquery`](https://github.com/hafenkran/duckdb-bigquery) - Integrates DuckDB with Google BigQuery, allowing direct querying and management of BigQuery datasets. - [`cache_httpfs`](https://github.com/dentiny/duck-read-cache-fs) - Adds a read caching layer to duckdb filesystem to improve query performance and reduce egress cost. - [`cache_prewarm`](https://github.com/dentiny/duckdb-cache-prewarm) - A Preloads table data blocks into the buffer pool or OS page cache, inspired by PostgreSQL's pg_prewarm extension. @@ -348,15 +348,19 @@ Community-contributed DuckDB extensions, which can be installed via `INSTALL ⟨ - [`dash`](https://github.com/gropaul/dash) - Fully local data canvas and dashboarding app within DuckDB. - [`duckherder`](https://github.com/dentiny/duckdb-distributed-execution) - Distributed execution for DuckDB queries. - [`duckpgq`](https://github.com/cwida/duckpgq-extension) - Add supports for SQL/PGQ (Property Graph Queries) introduced in the SQL:2023 standard. +- [`elasticsearch`](https://github.com/tlinhart/duckdb-elasticsearch) - Query Elasticsearch indices directly using SQL. - [`evalexpr_rhai`](https://github.com/rustyconover/duckdb-evalexpr-rhai-extension) - Evaluates the [Rhai](https://rhai.rs) scripting language as part of SQL. - [`fuzzycomplete`](https://github.com/rustyconover/duckdb-fuzzycomplete-extension) - Performs fuzzy string matching for autocompletion. +- [`gaggle`](https://github.com/CogitatorTech/gaggle) - A DuckDB extension for working with Kaggle datasets. - [`gsheets`](https://github.com/evidence-dev/duckdb_gsheets) - Read and write Google Sheets using SQL. - [`h3`](https://github.com/isaacbrodsky/h3-duckdb) - Adds support for the H3 discrete global grid system. -- [`httpserver`](https://github.com/quackscience/duckdb-extension-httpserver) - DuckDB HTTP API Server and Query Interface. - [`hostfs`](https://github.com/gropaul/hostfs) - Navigate and explore the local filesystem using SQL. +- [`httpserver`](https://github.com/quackscience/duckdb-extension-httpserver) - DuckDB HTTP API Server and Query Interface. +- [`infera`](https://github.com/CogitatorTech/infera) - A DuckDB extension for in-database inference. - [`lindel`](https://github.com/rustyconover/duckdb-lindel-extension) - Linearization/Delinearization, Z-Order, Hilbert and Morton Curves. - [`netquack`](https://github.com/hatamiarash7/duckdb-netquack) - Parsing, extracting, and analyzing domains, URIs, and paths with ease. - [`observefs`](https://github.com/dentiny/duckdb-filesystem-observability) - I/O observability for DuckDB filesystems with latency statistics and external file cache access insights. +- [`onager`](https://github.com/CogitatorTech/onager) - A DuckDB extension for graph data analytics. - [`pfc`](https://github.com/ImpossibleForge/pfc-duckdb) - Read block-indexed PFC-compressed JSONL logs with timestamp filtering — 25% smaller than gzip with minimal S3 egress. - [`prql`](https://github.com/ywelsch/duckdb-prql) - Run PRQL commands directly within DuckDB. - [`pst`](https://github.com/intellekthq/duckdb-pst) - Read Microsoft PST files in-place with rich schemas for emails, contacts, appointments, tasks, and more. @@ -365,10 +369,6 @@ Community-contributed DuckDB extensions, which can be installed via `INSTALL ⟨ - [`stats_duck`](https://github.com/KoliStat/the-stats-duck) - Statistics for tabular and clinical data: descriptive tables (`table_one`), linear models with robust/clustered standard errors, meta-analysis, bootstrap, and a grammar-of-graphics `VISUALIZE` clause that turns queries into Vega-Lite charts. - [`ulid`](https://github.com/Maxxen/duckdb_ulid) - ULID data type for DuckDB. A ULID is similar to a UUID except that it also contains a timestamp component. - [`yardstick`](https://github.com/sidequery/yardstick) - Implements Measures in SQL paper as a DuckDB extension for centralized metric definitions / en embedded semantic layer. -- [`infera`](https://github.com/CogitatorTech/infera) - A DuckDB extension for in-database inference. -- [`gaggle`](https://github.com/CogitatorTech/gaggle) - A DuckDB extension for working with Kaggle datasets. -- [`onager`](https://github.com/CogitatorTech/onager) - A DuckDB extension for graph data analytics. -- [`elasticsearch`](https://github.com/tlinhart/duckdb-elasticsearch) - Query Elasticsearch indices directly using SQL. ### Other Extensions