Improve README contribution checks (#327)

* Stabilize README link checks

* Enforce README entry ordering

* Ignore CI-blocked lib.rs links

.github/linkchecker/linkchecker.conf

@@ -1,15 +0,0 @@
-[output]
-log=html
-
-[checking]
-sslverify=0
-
-[filtering]
-checkextern=1
-ignorewarnings=http-redirected
-ignore=
-    https://harlequin\.sh.*
-    https://.*medium\.com.*
-    https://ibis-project\.org.*
-    https://enrich\.sh.*
-    https://streams\.dbconvert\.com.*

.github/lychee.toml

@@ -0,0 +1,16 @@
+no_progress = true
+max_retries = 3
+retry_wait_time = 2
+
+# These sites reject automated link checks even though the pages are available.
+exclude = [
+  '^https://harlequin\.sh',
+  '^https://.*medium\.com',
+  '^https://ibis-project\.org',
+  '^https://lib\.rs',
+  '^https://enrich\.sh',
+  '^https://streams\.dbconvert\.com',
+  '^https://communities\.sas\.com',
+  '^https://www\.packtpub\.com',
+  '^https://www\.transfermarkt\.co\.uk',
+]

.github/scripts/check-readme-order.mjs

@@ -0,0 +1,96 @@
+#!/usr/bin/env node
+
+import { readFileSync } from "node:fs";
+
+const [basePath, currentPath] = process.argv.slice(2);
+if (!basePath || !currentPath) {
+  console.error("Usage: check-readme-order.mjs BASE_README CURRENT_README");
+  process.exit(2);
+}
+
+const SORTED_SECTION = "Community Extensions";
+const IGNORED_SECTIONS = new Set(["Contents"]);
+
+function headingName(markdown) {
+  const linkedHeading = markdown.match(/^\[([^\]]+)\]\([^)]+\)$/);
+  return linkedHeading ? linkedHeading[1] : markdown;
+}
+
+function parseSections(path) {
+  const sections = new Map();
+  let section = null;
+
+  for (const line of readFileSync(path, "utf8").split("\n")) {
+    const heading = line.match(/^#{2,3}\s+(.+?)\s*$/);
+    if (heading) {
+      section = headingName(heading[1]);
+      if (!sections.has(section)) {
+        sections.set(section, []);
+      }
+      continue;
+    }
+
+    if (!section || !line.startsWith("- [")) {
+      continue;
+    }
+
+    const item = line.match(/^- \[(.*?)\]\(([^)\s]+)(?:\s+[^)]*)?\)/);
+    if (!item) {
+      continue;
+    }
+
+    const label = item[1].replaceAll("`", "").trim();
+    sections.get(section).push({
+      sortKey: label.toLowerCase(),
+      url: item[2],
+    });
+  }
+
+  return sections;
+}
+
+const baseSections = parseSections(basePath);
+const currentSections = parseSections(currentPath);
+let orderingIsValid = true;
+
+const sortedEntries = currentSections.get(SORTED_SECTION) ?? [];
+for (let index = 1; index < sortedEntries.length; index += 1) {
+  if (sortedEntries[index - 1].sortKey > sortedEntries[index].sortKey) {
+    orderingIsValid = false;
+  }
+}
+
+for (const [section, currentEntries] of currentSections) {
+  if (section === SORTED_SECTION || IGNORED_SECTIONS.has(section)) {
+    continue;
+  }
+
+  const baseEntries = baseSections.get(section) ?? [];
+  if (baseEntries.length === 0) {
+    continue;
+  }
+
+  const isExisting = (entry) =>
+    baseEntries.some(
+      (baseEntry) =>
+        baseEntry.url === entry.url || baseEntry.sortKey === entry.sortKey,
+    );
+
+  let foundNewEntry = false;
+  for (const entry of currentEntries) {
+    const entryAlreadyExists = isExisting(entry);
+    if (entryAlreadyExists && foundNewEntry) {
+      orderingIsValid = false;
+    } else if (!entryAlreadyExists) {
+      foundNewEntry = true;
+    }
+  }
+}
+
+if (!orderingIsValid) {
+  const message = "Thanks for contributing! Please move your new entry to follow the ordering rules in CONTRIBUTING.md, then update this pull request.";
+  console.error(`::error file=${currentPath},title=README entry order::${message}`);
+  process.exit(1);
+}
+
+console.log("README entries follow the required ordering.");

.github/workflows/linkchecker.yml

@@ -1,31 +1,46 @@
-name: Check links in the README
+name: Check the README
 
-on: [push, pull_request]
+on:
+  push:
+    branches: [main]
+    paths:
+      - README.md
+      - .github/lychee.toml
+      - .github/scripts/check-readme-order.mjs
+      - .github/workflows/linkchecker.yml
+  pull_request:
+    paths:
+      - README.md
+      - .github/lychee.toml
+      - .github/scripts/check-readme-order.mjs
+      - .github/workflows/linkchecker.yml
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: link-check-${{ github.ref }}
+  cancel-in-progress: true
 
 jobs:
   LinkChecker:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - name: Check entry order
+        if: github.event_name == 'pull_request'
+        env:
+          BASE_REF: ${{ github.base_ref }}
+        run: |
+          git fetch --no-tags --depth=1 origin "$BASE_REF"
+          git show FETCH_HEAD:README.md > "$RUNNER_TEMP/README.base.md"
+          node .github/scripts/check-readme-order.mjs "$RUNNER_TEMP/README.base.md" README.md
+
+      - name: Check links
+        uses: lycheeverse/lychee-action@e7477775783ea5526144ba13e8db5eec57747ce8 # v2.9.0
         with:
-          fetch-depth: 0
-
-      - uses: awalsh128/cache-apt-pkgs-action@latest
-        with:
-          packages: pandoc
-
-      - name: Convert Markdown to HTML format
-        run: pandoc README.md --standalone --output README.html
-
-      - name: Set up Python 3.9
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.9'
-
-      - run: pip install linkchecker --user
-
-      - name: Link Checker
-        run: linkchecker README.html --config .github/linkchecker/linkchecker.conf --file-output html/utf-8/report.html
-
-      - run: cat report.html >> ${GITHUB_STEP_SUMMARY}
-        if: success() || failure()
+          args: --config .github/lychee.toml README.md
+          lycheeVersion: v0.24.2

CONTRIBUTING.md

@@ -1,25 +1,28 @@
 # Contribution Guidelines
 
-Please note that this project is released with a
-[Contributor Code of Conduct](CODE_OF_CONDUCT.md). By participating in this
-project you agree to abide by its terms.
+By participating in this project, you agree to abide by its [Contributor Code of Conduct](CODE_OF_CONDUCT.md).
 
----
+## Adding an entry
 
-Ensure your pull request adheres to the following guidelines:
+- Check that the project or resource is not already listed.
+- Choose the most specific existing section.
+- Use the canonical project or repository URL.
+- Follow the surrounding Markdown format. Most entries use:
+  `- [Name](https://example.com) - A concise, factual description.`
+- Describe what the project does and how it relates to DuckDB. Avoid promotional claims.
+- Keep the pull request focused and disclose your affiliation when submitting your own project.
 
-- Make sure you take care of this
-- And this as well
-- And don't forget to check this
+### Entry order
 
-Thank you for your suggestions!
+Most sections preserve insertion order. **Append new entries to the bottom of the relevant list**, even when another entry seems similar. Do not alphabetize or otherwise reposition existing entries.
 
+The **Community Extensions** section is the exception: insert extensions in case-insensitive lexicographic order by extension name. Community extension names use backticks, for example:
+``- [`extension_name`](https://example.com) - A concise description.``
 
-## Updating your PR
+When updating an existing entry, leave it in its current position. When moving an entry to a different section, place it according to the destination section's rule.
 
-A lot of times, making a PR adhere to the standards above can be difficult.
-If the maintainers notice anything that we'd like changed, we'll ask you to
-edit your PR before we merge it. There's no need to open a new PR, just edit
-the existing one. If you're not sure how to do that,
-[here is a guide](https://github.com/RichardLitt/knowledge/blob/master/github/amending-a-commit-guide.md)
-on the different ways you can update your PR so that we can merge it.
+## Before opening a pull request
+
+- Check the Markdown diff for unrelated changes.
+- Verify that every new URL works without authentication.
+- Confirm that each new entry is appended, or lexicographically placed when it is a Community Extension.

README.md

@@ -337,8 +337,8 @@ Official DuckDB extensions, which can installed via `INSTALL ⟨extension_name
 
 Community-contributed DuckDB extensions, which can be installed via `INSTALL ⟨extension_name⟩ FROM community`.
 
-- [`acp`](https://github.com/sidequery/duckdb-acp) - Embeds AI agents such as Claude Code inside of DuckDB via Agent Client Protocol.
 - [`3fs`](https://github.com/open3fs/duckdb-3fs) - Integrates DuckDB with DeepSeek 3FS distributed file system.
+- [`acp`](https://github.com/sidequery/duckdb-acp) - Embeds AI agents such as Claude Code inside of DuckDB via Agent Client Protocol.
 - [`bigquery`](https://github.com/hafenkran/duckdb-bigquery) - Integrates DuckDB with Google BigQuery, allowing direct querying and management of BigQuery datasets.
 - [`cache_httpfs`](https://github.com/dentiny/duck-read-cache-fs) - Adds a read caching layer to duckdb filesystem to improve query performance and reduce egress cost.
 - [`cache_prewarm`](https://github.com/dentiny/duckdb-cache-prewarm) - A Preloads table data blocks into the buffer pool or OS page cache, inspired by PostgreSQL's pg_prewarm extension.
@@ -348,15 +348,19 @@ Community-contributed DuckDB extensions, which can be installed via `INSTALL ⟨
 - [`dash`](https://github.com/gropaul/dash) - Fully local data canvas and dashboarding app within DuckDB.
 - [`duckherder`](https://github.com/dentiny/duckdb-distributed-execution) - Distributed execution for DuckDB queries.
 - [`duckpgq`](https://github.com/cwida/duckpgq-extension) - Add supports for SQL/PGQ (Property Graph Queries) introduced in the SQL:2023 standard.
+- [`elasticsearch`](https://github.com/tlinhart/duckdb-elasticsearch) - Query Elasticsearch indices directly using SQL.
 - [`evalexpr_rhai`](https://github.com/rustyconover/duckdb-evalexpr-rhai-extension) - Evaluates the [Rhai](https://rhai.rs) scripting language as part of SQL.
 - [`fuzzycomplete`](https://github.com/rustyconover/duckdb-fuzzycomplete-extension) - Performs fuzzy string matching for autocompletion.
+- [`gaggle`](https://github.com/CogitatorTech/gaggle) - A DuckDB extension for working with Kaggle datasets.
 - [`gsheets`](https://github.com/evidence-dev/duckdb_gsheets) - Read and write Google Sheets using SQL.
 - [`h3`](https://github.com/isaacbrodsky/h3-duckdb) - Adds support for the H3 discrete global grid system.
-- [`httpserver`](https://github.com/quackscience/duckdb-extension-httpserver) - DuckDB HTTP API Server and Query Interface.
 - [`hostfs`](https://github.com/gropaul/hostfs) - Navigate and explore the local filesystem using SQL.
+- [`httpserver`](https://github.com/quackscience/duckdb-extension-httpserver) - DuckDB HTTP API Server and Query Interface.
+- [`infera`](https://github.com/CogitatorTech/infera) - A DuckDB extension for in-database inference.
 - [`lindel`](https://github.com/rustyconover/duckdb-lindel-extension) - Linearization/Delinearization, Z-Order, Hilbert and Morton Curves.
 - [`netquack`](https://github.com/hatamiarash7/duckdb-netquack) - Parsing, extracting, and analyzing domains, URIs, and paths with ease.
 - [`observefs`](https://github.com/dentiny/duckdb-filesystem-observability) - I/O observability for DuckDB filesystems with latency statistics and external file cache access insights.
+- [`onager`](https://github.com/CogitatorTech/onager) - A DuckDB extension for graph data analytics.
 - [`pfc`](https://github.com/ImpossibleForge/pfc-duckdb) - Read block-indexed PFC-compressed JSONL logs with timestamp filtering — 25% smaller than gzip with minimal S3 egress.
 - [`prql`](https://github.com/ywelsch/duckdb-prql) - Run PRQL commands directly within DuckDB.
 - [`pst`](https://github.com/intellekthq/duckdb-pst) - Read Microsoft PST files in-place with rich schemas for emails, contacts, appointments, tasks, and more.
@@ -365,10 +369,6 @@ Community-contributed DuckDB extensions, which can be installed via `INSTALL ⟨
 - [`stats_duck`](https://github.com/KoliStat/the-stats-duck) - Statistics for tabular and clinical data: descriptive tables (`table_one`), linear models with robust/clustered standard errors, meta-analysis, bootstrap, and a grammar-of-graphics `VISUALIZE` clause that turns queries into Vega-Lite charts.
 - [`ulid`](https://github.com/Maxxen/duckdb_ulid) - ULID data type for DuckDB. A ULID is similar to a UUID except that it also contains a timestamp component.
 - [`yardstick`](https://github.com/sidequery/yardstick) - Implements Measures in SQL paper as a DuckDB extension for centralized metric definitions / en embedded semantic layer.
-- [`infera`](https://github.com/CogitatorTech/infera) - A DuckDB extension for in-database inference.
-- [`gaggle`](https://github.com/CogitatorTech/gaggle) - A DuckDB extension for working with Kaggle datasets.
-- [`onager`](https://github.com/CogitatorTech/onager) - A DuckDB extension for graph data analytics.
-- [`elasticsearch`](https://github.com/tlinhart/duckdb-elasticsearch) - Query Elasticsearch indices directly using SQL.
 
 ### Other Extensions