mirror of
https://github.com/davidgasquez/awesome-duckdb.git
synced 2026-11-18 08:43:41 +00:00
Improve README contribution checks (#327)
* Stabilize README link checks * Enforce README entry ordering * Ignore CI-blocked lib.rs links
This commit is contained in:
1 parent
16b7d79974
commit
009e09359b
6 files changed
+177
-62
No files matched your search
@@ -1,15 +0,0 @@
|
|||||||
[output]
|
|
||||||
log=html
|
|
||||||
|
|
||||||
[checking]
|
|
||||||
sslverify=0
|
|
||||||
|
|
||||||
[filtering]
|
|
||||||
checkextern=1
|
|
||||||
ignorewarnings=http-redirected
|
|
||||||
ignore=
|
|
||||||
https://harlequin\.sh.*
|
|
||||||
https://.*medium\.com.*
|
|
||||||
https://ibis-project\.org.*
|
|
||||||
https://enrich\.sh.*
|
|
||||||
https://streams\.dbconvert\.com.*
|
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
no_progress = true
|
||||||
|
max_retries = 3
|
||||||
|
retry_wait_time = 2
|
||||||
|
|
||||||
|
# These sites reject automated link checks even though the pages are available.
|
||||||
|
exclude = [
|
||||||
|
'^https://harlequin\.sh',
|
||||||
|
'^https://.*medium\.com',
|
||||||
|
'^https://ibis-project\.org',
|
||||||
|
'^https://lib\.rs',
|
||||||
|
'^https://enrich\.sh',
|
||||||
|
'^https://streams\.dbconvert\.com',
|
||||||
|
'^https://communities\.sas\.com',
|
||||||
|
'^https://www\.packtpub\.com',
|
||||||
|
'^https://www\.transfermarkt\.co\.uk',
|
||||||
|
]
|
||||||
@@ -0,0 +1,96 @@
|
|||||||
|
#!/usr/bin/env node
|
||||||
|
|
||||||
|
import { readFileSync } from "node:fs";
|
||||||
|
|
||||||
|
const [basePath, currentPath] = process.argv.slice(2);
|
||||||
|
if (!basePath || !currentPath) {
|
||||||
|
console.error("Usage: check-readme-order.mjs BASE_README CURRENT_README");
|
||||||
|
process.exit(2);
|
||||||
|
}
|
||||||
|
|
||||||
|
const SORTED_SECTION = "Community Extensions";
|
||||||
|
const IGNORED_SECTIONS = new Set(["Contents"]);
|
||||||
|
|
||||||
|
function headingName(markdown) {
|
||||||
|
const linkedHeading = markdown.match(/^\[([^\]]+)\]\([^)]+\)$/);
|
||||||
|
return linkedHeading ? linkedHeading[1] : markdown;
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseSections(path) {
|
||||||
|
const sections = new Map();
|
||||||
|
let section = null;
|
||||||
|
|
||||||
|
for (const line of readFileSync(path, "utf8").split("\n")) {
|
||||||
|
const heading = line.match(/^#{2,3}\s+(.+?)\s*$/);
|
||||||
|
if (heading) {
|
||||||
|
section = headingName(heading[1]);
|
||||||
|
if (!sections.has(section)) {
|
||||||
|
sections.set(section, []);
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!section || !line.startsWith("- [")) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const item = line.match(/^- \[(.*?)\]\(([^)\s]+)(?:\s+[^)]*)?\)/);
|
||||||
|
if (!item) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const label = item[1].replaceAll("`", "").trim();
|
||||||
|
sections.get(section).push({
|
||||||
|
sortKey: label.toLowerCase(),
|
||||||
|
url: item[2],
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
return sections;
|
||||||
|
}
|
||||||
|
|
||||||
|
const baseSections = parseSections(basePath);
|
||||||
|
const currentSections = parseSections(currentPath);
|
||||||
|
let orderingIsValid = true;
|
||||||
|
|
||||||
|
const sortedEntries = currentSections.get(SORTED_SECTION) ?? [];
|
||||||
|
for (let index = 1; index < sortedEntries.length; index += 1) {
|
||||||
|
if (sortedEntries[index - 1].sortKey > sortedEntries[index].sortKey) {
|
||||||
|
orderingIsValid = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const [section, currentEntries] of currentSections) {
|
||||||
|
if (section === SORTED_SECTION || IGNORED_SECTIONS.has(section)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const baseEntries = baseSections.get(section) ?? [];
|
||||||
|
if (baseEntries.length === 0) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const isExisting = (entry) =>
|
||||||
|
baseEntries.some(
|
||||||
|
(baseEntry) =>
|
||||||
|
baseEntry.url === entry.url || baseEntry.sortKey === entry.sortKey,
|
||||||
|
);
|
||||||
|
|
||||||
|
let foundNewEntry = false;
|
||||||
|
for (const entry of currentEntries) {
|
||||||
|
const entryAlreadyExists = isExisting(entry);
|
||||||
|
if (entryAlreadyExists && foundNewEntry) {
|
||||||
|
orderingIsValid = false;
|
||||||
|
} else if (!entryAlreadyExists) {
|
||||||
|
foundNewEntry = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!orderingIsValid) {
|
||||||
|
const message = "Thanks for contributing! Please move your new entry to follow the ordering rules in CONTRIBUTING.md, then update this pull request.";
|
||||||
|
console.error(`::error file=${currentPath},title=README entry order::${message}`);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log("README entries follow the required ordering.");
|
||||||
@@ -1,31 +1,46 @@
|
|||||||
name: Check links in the README
|
name: Check the README
|
||||||
|
|
||||||
on: [push, pull_request]
|
on:
|
||||||
|
push:
|
||||||
|
branches: [main]
|
||||||
|
paths:
|
||||||
|
- README.md
|
||||||
|
- .github/lychee.toml
|
||||||
|
- .github/scripts/check-readme-order.mjs
|
||||||
|
- .github/workflows/linkchecker.yml
|
||||||
|
pull_request:
|
||||||
|
paths:
|
||||||
|
- README.md
|
||||||
|
- .github/lychee.toml
|
||||||
|
- .github/scripts/check-readme-order.mjs
|
||||||
|
- .github/workflows/linkchecker.yml
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: link-check-${{ github.ref }}
|
||||||
|
cancel-in-progress: true
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
LinkChecker:
|
LinkChecker:
|
||||||
runs-on: ubuntu-22.04
|
runs-on: ubuntu-latest
|
||||||
|
timeout-minutes: 5
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v3
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||||
|
|
||||||
|
- name: Check entry order
|
||||||
|
if: github.event_name == 'pull_request'
|
||||||
|
env:
|
||||||
|
BASE_REF: ${{ github.base_ref }}
|
||||||
|
run: |
|
||||||
|
git fetch --no-tags --depth=1 origin "$BASE_REF"
|
||||||
|
git show FETCH_HEAD:README.md > "$RUNNER_TEMP/README.base.md"
|
||||||
|
node .github/scripts/check-readme-order.mjs "$RUNNER_TEMP/README.base.md" README.md
|
||||||
|
|
||||||
|
- name: Check links
|
||||||
|
uses: lycheeverse/lychee-action@e7477775783ea5526144ba13e8db5eec57747ce8 # v2.9.0
|
||||||
with:
|
with:
|
||||||
fetch-depth: 0
|
args: --config .github/lychee.toml README.md
|
||||||
|
lycheeVersion: v0.24.2
|
||||||
- uses: awalsh128/cache-apt-pkgs-action@latest
|
|
||||||
with:
|
|
||||||
packages: pandoc
|
|
||||||
|
|
||||||
- name: Convert Markdown to HTML format
|
|
||||||
run: pandoc README.md --standalone --output README.html
|
|
||||||
|
|
||||||
- name: Set up Python 3.9
|
|
||||||
uses: actions/setup-python@v4
|
|
||||||
with:
|
|
||||||
python-version: '3.9'
|
|
||||||
|
|
||||||
- run: pip install linkchecker --user
|
|
||||||
|
|
||||||
- name: Link Checker
|
|
||||||
run: linkchecker README.html --config .github/linkchecker/linkchecker.conf --file-output html/utf-8/report.html
|
|
||||||
|
|
||||||
- run: cat report.html >> ${GITHUB_STEP_SUMMARY}
|
|
||||||
if: success() || failure()
|
|
||||||
+19
-16
@@ -1,25 +1,28 @@
|
|||||||
# Contribution Guidelines
|
# Contribution Guidelines
|
||||||
|
|
||||||
Please note that this project is released with a
|
By participating in this project, you agree to abide by its [Contributor Code of Conduct](CODE_OF_CONDUCT.md).
|
||||||
[Contributor Code of Conduct](CODE_OF_CONDUCT.md). By participating in this
|
|
||||||
project you agree to abide by its terms.
|
|
||||||
|
|
||||||
---
|
## Adding an entry
|
||||||
|
|
||||||
Ensure your pull request adheres to the following guidelines:
|
- Check that the project or resource is not already listed.
|
||||||
|
- Choose the most specific existing section.
|
||||||
|
- Use the canonical project or repository URL.
|
||||||
|
- Follow the surrounding Markdown format. Most entries use:
|
||||||
|
`- [Name](https://example.com) - A concise, factual description.`
|
||||||
|
- Describe what the project does and how it relates to DuckDB. Avoid promotional claims.
|
||||||
|
- Keep the pull request focused and disclose your affiliation when submitting your own project.
|
||||||
|
|
||||||
- Make sure you take care of this
|
### Entry order
|
||||||
- And this as well
|
|
||||||
- And don't forget to check this
|
|
||||||
|
|
||||||
Thank you for your suggestions!
|
Most sections preserve insertion order. **Append new entries to the bottom of the relevant list**, even when another entry seems similar. Do not alphabetize or otherwise reposition existing entries.
|
||||||
|
|
||||||
|
The **Community Extensions** section is the exception: insert extensions in case-insensitive lexicographic order by extension name. Community extension names use backticks, for example:
|
||||||
|
``- [`extension_name`](https://example.com) - A concise description.``
|
||||||
|
|
||||||
## Updating your PR
|
When updating an existing entry, leave it in its current position. When moving an entry to a different section, place it according to the destination section's rule.
|
||||||
|
|
||||||
A lot of times, making a PR adhere to the standards above can be difficult.
|
## Before opening a pull request
|
||||||
If the maintainers notice anything that we'd like changed, we'll ask you to
|
|
||||||
edit your PR before we merge it. There's no need to open a new PR, just edit
|
- Check the Markdown diff for unrelated changes.
|
||||||
the existing one. If you're not sure how to do that,
|
- Verify that every new URL works without authentication.
|
||||||
[here is a guide](https://github.com/RichardLitt/knowledge/blob/master/github/amending-a-commit-guide.md)
|
- Confirm that each new entry is appended, or lexicographically placed when it is a Community Extension.
|
||||||
on the different ways you can update your PR so that we can merge it.
|
|
||||||
@@ -337,8 +337,8 @@ Official DuckDB extensions, which can installed via `INSTALL ⟨extension_name
|
|||||||
|
|
||||||
Community-contributed DuckDB extensions, which can be installed via `INSTALL ⟨extension_name⟩ FROM community`.
|
Community-contributed DuckDB extensions, which can be installed via `INSTALL ⟨extension_name⟩ FROM community`.
|
||||||
|
|
||||||
- [`acp`](https://github.com/sidequery/duckdb-acp) - Embeds AI agents such as Claude Code inside of DuckDB via Agent Client Protocol.
|
|
||||||
- [`3fs`](https://github.com/open3fs/duckdb-3fs) - Integrates DuckDB with DeepSeek 3FS distributed file system.
|
- [`3fs`](https://github.com/open3fs/duckdb-3fs) - Integrates DuckDB with DeepSeek 3FS distributed file system.
|
||||||
|
- [`acp`](https://github.com/sidequery/duckdb-acp) - Embeds AI agents such as Claude Code inside of DuckDB via Agent Client Protocol.
|
||||||
- [`bigquery`](https://github.com/hafenkran/duckdb-bigquery) - Integrates DuckDB with Google BigQuery, allowing direct querying and management of BigQuery datasets.
|
- [`bigquery`](https://github.com/hafenkran/duckdb-bigquery) - Integrates DuckDB with Google BigQuery, allowing direct querying and management of BigQuery datasets.
|
||||||
- [`cache_httpfs`](https://github.com/dentiny/duck-read-cache-fs) - Adds a read caching layer to duckdb filesystem to improve query performance and reduce egress cost.
|
- [`cache_httpfs`](https://github.com/dentiny/duck-read-cache-fs) - Adds a read caching layer to duckdb filesystem to improve query performance and reduce egress cost.
|
||||||
- [`cache_prewarm`](https://github.com/dentiny/duckdb-cache-prewarm) - A Preloads table data blocks into the buffer pool or OS page cache, inspired by PostgreSQL's pg_prewarm extension.
|
- [`cache_prewarm`](https://github.com/dentiny/duckdb-cache-prewarm) - A Preloads table data blocks into the buffer pool or OS page cache, inspired by PostgreSQL's pg_prewarm extension.
|
||||||
@@ -348,15 +348,19 @@ Community-contributed DuckDB extensions, which can be installed via `INSTALL ⟨
|
|||||||
- [`dash`](https://github.com/gropaul/dash) - Fully local data canvas and dashboarding app within DuckDB.
|
- [`dash`](https://github.com/gropaul/dash) - Fully local data canvas and dashboarding app within DuckDB.
|
||||||
- [`duckherder`](https://github.com/dentiny/duckdb-distributed-execution) - Distributed execution for DuckDB queries.
|
- [`duckherder`](https://github.com/dentiny/duckdb-distributed-execution) - Distributed execution for DuckDB queries.
|
||||||
- [`duckpgq`](https://github.com/cwida/duckpgq-extension) - Add supports for SQL/PGQ (Property Graph Queries) introduced in the SQL:2023 standard.
|
- [`duckpgq`](https://github.com/cwida/duckpgq-extension) - Add supports for SQL/PGQ (Property Graph Queries) introduced in the SQL:2023 standard.
|
||||||
|
- [`elasticsearch`](https://github.com/tlinhart/duckdb-elasticsearch) - Query Elasticsearch indices directly using SQL.
|
||||||
- [`evalexpr_rhai`](https://github.com/rustyconover/duckdb-evalexpr-rhai-extension) - Evaluates the [Rhai](https://rhai.rs) scripting language as part of SQL.
|
- [`evalexpr_rhai`](https://github.com/rustyconover/duckdb-evalexpr-rhai-extension) - Evaluates the [Rhai](https://rhai.rs) scripting language as part of SQL.
|
||||||
- [`fuzzycomplete`](https://github.com/rustyconover/duckdb-fuzzycomplete-extension) - Performs fuzzy string matching for autocompletion.
|
- [`fuzzycomplete`](https://github.com/rustyconover/duckdb-fuzzycomplete-extension) - Performs fuzzy string matching for autocompletion.
|
||||||
|
- [`gaggle`](https://github.com/CogitatorTech/gaggle) - A DuckDB extension for working with Kaggle datasets.
|
||||||
- [`gsheets`](https://github.com/evidence-dev/duckdb_gsheets) - Read and write Google Sheets using SQL.
|
- [`gsheets`](https://github.com/evidence-dev/duckdb_gsheets) - Read and write Google Sheets using SQL.
|
||||||
- [`h3`](https://github.com/isaacbrodsky/h3-duckdb) - Adds support for the H3 discrete global grid system.
|
- [`h3`](https://github.com/isaacbrodsky/h3-duckdb) - Adds support for the H3 discrete global grid system.
|
||||||
- [`httpserver`](https://github.com/quackscience/duckdb-extension-httpserver) - DuckDB HTTP API Server and Query Interface.
|
|
||||||
- [`hostfs`](https://github.com/gropaul/hostfs) - Navigate and explore the local filesystem using SQL.
|
- [`hostfs`](https://github.com/gropaul/hostfs) - Navigate and explore the local filesystem using SQL.
|
||||||
|
- [`httpserver`](https://github.com/quackscience/duckdb-extension-httpserver) - DuckDB HTTP API Server and Query Interface.
|
||||||
|
- [`infera`](https://github.com/CogitatorTech/infera) - A DuckDB extension for in-database inference.
|
||||||
- [`lindel`](https://github.com/rustyconover/duckdb-lindel-extension) - Linearization/Delinearization, Z-Order, Hilbert and Morton Curves.
|
- [`lindel`](https://github.com/rustyconover/duckdb-lindel-extension) - Linearization/Delinearization, Z-Order, Hilbert and Morton Curves.
|
||||||
- [`netquack`](https://github.com/hatamiarash7/duckdb-netquack) - Parsing, extracting, and analyzing domains, URIs, and paths with ease.
|
- [`netquack`](https://github.com/hatamiarash7/duckdb-netquack) - Parsing, extracting, and analyzing domains, URIs, and paths with ease.
|
||||||
- [`observefs`](https://github.com/dentiny/duckdb-filesystem-observability) - I/O observability for DuckDB filesystems with latency statistics and external file cache access insights.
|
- [`observefs`](https://github.com/dentiny/duckdb-filesystem-observability) - I/O observability for DuckDB filesystems with latency statistics and external file cache access insights.
|
||||||
|
- [`onager`](https://github.com/CogitatorTech/onager) - A DuckDB extension for graph data analytics.
|
||||||
- [`pfc`](https://github.com/ImpossibleForge/pfc-duckdb) - Read block-indexed PFC-compressed JSONL logs with timestamp filtering — 25% smaller than gzip with minimal S3 egress.
|
- [`pfc`](https://github.com/ImpossibleForge/pfc-duckdb) - Read block-indexed PFC-compressed JSONL logs with timestamp filtering — 25% smaller than gzip with minimal S3 egress.
|
||||||
- [`prql`](https://github.com/ywelsch/duckdb-prql) - Run PRQL commands directly within DuckDB.
|
- [`prql`](https://github.com/ywelsch/duckdb-prql) - Run PRQL commands directly within DuckDB.
|
||||||
- [`pst`](https://github.com/intellekthq/duckdb-pst) - Read Microsoft PST files in-place with rich schemas for emails, contacts, appointments, tasks, and more.
|
- [`pst`](https://github.com/intellekthq/duckdb-pst) - Read Microsoft PST files in-place with rich schemas for emails, contacts, appointments, tasks, and more.
|
||||||
@@ -365,10 +369,6 @@ Community-contributed DuckDB extensions, which can be installed via `INSTALL ⟨
|
|||||||
- [`stats_duck`](https://github.com/KoliStat/the-stats-duck) - Statistics for tabular and clinical data: descriptive tables (`table_one`), linear models with robust/clustered standard errors, meta-analysis, bootstrap, and a grammar-of-graphics `VISUALIZE` clause that turns queries into Vega-Lite charts.
|
- [`stats_duck`](https://github.com/KoliStat/the-stats-duck) - Statistics for tabular and clinical data: descriptive tables (`table_one`), linear models with robust/clustered standard errors, meta-analysis, bootstrap, and a grammar-of-graphics `VISUALIZE` clause that turns queries into Vega-Lite charts.
|
||||||
- [`ulid`](https://github.com/Maxxen/duckdb_ulid) - ULID data type for DuckDB. A ULID is similar to a UUID except that it also contains a timestamp component.
|
- [`ulid`](https://github.com/Maxxen/duckdb_ulid) - ULID data type for DuckDB. A ULID is similar to a UUID except that it also contains a timestamp component.
|
||||||
- [`yardstick`](https://github.com/sidequery/yardstick) - Implements Measures in SQL paper as a DuckDB extension for centralized metric definitions / en embedded semantic layer.
|
- [`yardstick`](https://github.com/sidequery/yardstick) - Implements Measures in SQL paper as a DuckDB extension for centralized metric definitions / en embedded semantic layer.
|
||||||
- [`infera`](https://github.com/CogitatorTech/infera) - A DuckDB extension for in-database inference.
|
|
||||||
- [`gaggle`](https://github.com/CogitatorTech/gaggle) - A DuckDB extension for working with Kaggle datasets.
|
|
||||||
- [`onager`](https://github.com/CogitatorTech/onager) - A DuckDB extension for graph data analytics.
|
|
||||||
- [`elasticsearch`](https://github.com/tlinhart/duckdb-elasticsearch) - Query Elasticsearch indices directly using SQL.
|
|
||||||
|
|
||||||
### Other Extensions
|
### Other Extensions
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user