polo/baseldosdados
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update infrastructure and auth configuration, add package management files

Browse Source
This commit is contained in:
rafapolo
2026-04-15 15:20:10 +02:00
parent 6bc2a0608a
commit 86a1669902
10 changed files with 1650 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@@ -7,5 +7,6 @@ done_transfers.txt
**/target
*.log
.venv
node_modules/
.cocoindex_code
.cocoindex_code

18
Caddyfile
View File

@@ -17,6 +17,15 @@
}
handle /query {
header Access-Control-Allow-Origin *
header Access-Control-Allow-Methods "GET, POST, OPTIONS"
header Access-Control-Allow-Headers "X-Password, Content-Type"
@options method OPTIONS
handle @options {
respond 204
}
reverse_proxy 127.0.0.1:8081
}
@@ -52,6 +61,15 @@
}
handle /query {
header Access-Control-Allow-Origin *
header Access-Control-Allow-Methods "GET, POST, OPTIONS"
header Access-Control-Allow-Headers "X-Password, Content-Type"
@options method OPTIONS
handle @options {
respond 204
}
reverse_proxy 127.0.0.1:8081
}

22
Dockerfile
View File

@@ -13,18 +13,27 @@ RUN apt-get update -qq && \
rm /tmp/libduckdb.zip && \
apt-get clean && rm -rf /var/lib/apt/lists/*
WORKDIR /build
WORKDIR /build/ask
COPY ask/Cargo.toml ask/Cargo.lock ./
COPY ask/src ./src
RUN rustup target add x86_64-unknown-linux-musl && \
mkdir -p /build/.cargo && \
printf '[target.x86_64-unknown-linux-musl]\nlinker = "gcc"\n' > /build/.cargo/config.toml && \
printf 'rustflags = ["-L", "/usr/local/lib", "-C", "target-feature=+crt-static"]\n' >> /build/.cargo/config.toml && \
mkdir -p /build/ask/.cargo && \
printf '[target.x86_64-unknown-linux-musl]\nlinker = "gcc"\n' > /build/ask/.cargo/config.toml && \
printf 'rustflags = ["-L", "/usr/local/lib", "-C", "target-feature=+crt-static"]\n' >> /build/ask/.cargo/config.toml && \
sed -i 's/features = \["bundled"\]/default-features = false/' Cargo.toml && \
rm -f Cargo.lock && \
cargo build --release --target x86_64-unknown-linux-musl
WORKDIR /build/dbquery
COPY dbquery/Cargo.toml dbquery/Cargo.lock* ./
COPY dbquery/src ./src
RUN mkdir -p /build/dbquery/.cargo && \
printf '[target.x86_64-unknown-linux-musl]\nlinker = "gcc"\n' > /build/dbquery/.cargo/config.toml && \
printf 'rustflags = ["-C", "target-feature=+crt-static"]\n' >> /build/dbquery/.cargo/config.toml && \
cargo build --release --target x86_64-unknown-linux-musl
FROM --platform=linux/amd64 debian:12-slim
RUN apt-get update -qq && \
@@ -54,7 +63,8 @@ RUN apt-get update -qq && \
WORKDIR /app
COPY --from=builder /build/target/x86_64-unknown-linux-musl/release/ask ./ask
COPY --from=builder /build/ask/target/x86_64-unknown-linux-musl/release/ask ./ask
COPY --from=builder /build/dbquery/target/x86_64-unknown-linux-musl/release/dbquery ./dbquery
COPY ask/system_prompt.md ./system_prompt.md
COPY data/basedosdados.duckdb ./data/
COPY context ./context/
@@ -62,7 +72,7 @@ COPY auth.py ./
COPY start.sh ./
COPY Caddyfile ./
RUN chmod +x start.sh ask
RUN chmod +x start.sh ask dbquery
EXPOSE 8080

55
auth.py
View File

@@ -1,10 +1,32 @@
#!/usr/bin/env python3
"""Minimal cookie-session auth gate for DuckDB shell."""
import hmac, hashlib, os, secrets, subprocess, time
import hmac, hashlib, json, os, secrets, subprocess, time
from http.server import HTTPServer, BaseHTTPRequestHandler
from socketserver import ThreadingMixIn
from urllib.parse import parse_qs
PASSWORD = os.environ.get('BASIC_AUTH_PASSWORD', '').encode()
_INIT_SQL = None
def _run_query(sql, json_mode=True):
global _INIT_SQL
if _INIT_SQL is None:
with open('/app/ssh_init.sql') as f:
_INIT_SQL = f.read()
if json_mode:
sql = '.mode json\n' + sql
try:
r = subprocess.run(
['duckdb', '-readonly', '/app/data/basedosdados.duckdb'],
input=_INIT_SQL + '\n' + sql, capture_output=True, text=True, timeout=120
)
if r.stdout.strip().startswith('['):
return r.stdout.encode()
err = (r.stderr or r.stdout or 'unknown DuckDB error').strip()
return json.dumps({'error': err}).encode()
except subprocess.TimeoutExpired:
return json.dumps({'error': 'query timed out after 120s'}).encode()
_SECRET = secrets.token_bytes(32)
def _make_token():
@@ -51,6 +73,18 @@ class H(BaseHTTPRequestHandler):
self.send_response(302)
self.send_header('Location', '/login')
self.end_headers()
elif self.path.startswith('/query'):
from urllib.parse import urlparse, parse_qs
pwd = self.headers.get('X-Password', '').encode()
if not hmac.compare_digest(pwd, PASSWORD):
self._resp(401, b'Unauthorized\n')
return
qs = parse_qs(urlparse(self.path).query)
sql = qs.get('q', [''])[0]
if not sql:
self._resp(400, b'missing ?q=\n')
return
self._resp(200, _run_query(sql), 'application/json')
else:
self._resp(200, LOGIN_HTML, 'text/html; charset=utf-8')
@@ -61,15 +95,7 @@ class H(BaseHTTPRequestHandler):
self._resp(401, b'Unauthorized\n')
return
sql = self.rfile.read(int(self.headers.get('Content-Length', 0))).decode(errors='replace')
try:
r = subprocess.run(
['duckdb', '-readonly', '--init', '/app/ssh_init.sql', '/app/basedosdados.duckdb'],
input=sql, capture_output=True, text=True, timeout=120
)
out = (r.stdout + r.stderr).encode()
except subprocess.TimeoutExpired:
out = b'timeout\n'
self._resp(200, out, 'text/plain; charset=utf-8')
self._resp(200, _run_query(sql), 'application/json')
return
body = self.rfile.read(int(self.headers.get('Content-Length', 0))).decode(errors='replace')
pwd = parse_qs(body).get('password', [''])[0].encode()
@@ -81,6 +107,10 @@ class H(BaseHTTPRequestHandler):
else:
self._resp(200, LOGIN_HTML, 'text/html; charset=utf-8')
def do_OPTIONS(self):
self.send_response(204)
self.end_headers()
def _resp(self, code, body=b'', ct='text/plain'):
self.send_response(code)
if body:
@@ -93,4 +123,7 @@ class H(BaseHTTPRequestHandler):
def log_message(self, *_):
pass
HTTPServer(('127.0.0.1', 8081), H).serve_forever()
class ThreadedHTTPServer(ThreadingMixIn, HTTPServer):
daemon_threads = True
ThreadedHTTPServer(('127.0.0.1', 8081), H).serve_forever()

1430
dbquery/Cargo.lock generated Normal file
View File

File diff suppressed because it is too large Load Diff

8
dbquery/Cargo.toml Normal file
View File

@@ -0,0 +1,8 @@
[package]
name = "dbquery"
version = "0.1.0"
edition = "2024"
[dependencies]
reqwest = { version = "0.12", features = ["blocking", "rustls-tls"], default-features = false }
clap = { version = "4", features = ["derive", "env"] }

68
dbquery/src/main.rs Normal file
View File

@@ -0,0 +1,68 @@
use clap::{Parser, ValueEnum};
use std::io::{self, Read};
#[derive(Clone, ValueEnum)]
enum Method {
Get,
Post,
}
#[derive(Parser)]
#[command(about = "Query a db endpoint (always returns JSON)")]
struct Args {
/// SQL query (if omitted, reads from stdin)
query: Option<String>,
/// Base server URL
#[arg(short, long, default_value = "https://db.xn--2dk.xyz")]
server: String,
/// API password (or set BASIC_AUTH_PASSWORD env var)
#[arg(short, long, env = "BASIC_AUTH_PASSWORD")]
password: String,
/// HTTP method: GET sends query as ?q= param, POST sends as body
#[arg(short = 'X', long, value_enum, default_value = "post")]
method: Method,
}
fn main() {
let args = Args::parse();
let query = match args.query {
Some(q) => q,
None => {
let mut buf = String::new();
io::stdin().read_to_string(&mut buf).expect("failed to read stdin");
buf
}
};
let base = format!("{}/query", args.server.trim_end_matches('/'));
let client = reqwest::blocking::Client::new();
let resp = match args.method {
Method::Get => client
.get(&base)
.header("X-Password", &args.password)
.query(&[("q", &query)])
.send()
.expect("request failed"),
Method::Post => client
.post(&base)
.header("X-Password", &args.password)
.body(query)
.send()
.expect("request failed"),
};
let status = resp.status();
let body = resp.text().expect("failed to read response");
if !status.is_success() {
eprintln!("Error {status}: {body}");
std::process::exit(1);
}
print!("{body}");
}

59
package-lock.json generated Normal file
View File

@@ -0,0 +1,59 @@
{
"name": "baseldosdados",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"devDependencies": {
"playwright": "^1.59.1"
}
},
"node_modules/fsevents": {
"version": "2.3.2",
"resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
"integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
"dev": true,
"hasInstallScript": true,
"license": "MIT",
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": "^8.16.0 || ^10.6.0 || >=11.0.0"
}
},
"node_modules/playwright": {
"version": "1.59.1",
"resolved": "https://registry.npmjs.org/playwright/-/playwright-1.59.1.tgz",
"integrity": "sha512-C8oWjPR3F81yljW9o5OxcWzfh6avkVwDD2VYdwIGqTkl+OGFISgypqzfu7dOe4QNLL2aqcWBmI3PMtLIK233lw==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
"playwright-core": "1.59.1"
},
"bin": {
"playwright": "cli.js"
},
"engines": {
"node": ">=18"
},
"optionalDependencies": {
"fsevents": "2.3.2"
}
},
"node_modules/playwright-core": {
"version": "1.59.1",
"resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.59.1.tgz",
"integrity": "sha512-HBV/RJg81z5BiiZ9yPzIiClYV/QMsDCKUyogwH9p3MCP6IYjUFu/MActgYAvK0oWyV9NlwM3GLBjADyWgydVyg==",
"dev": true,
"license": "Apache-2.0",
"bin": {
"playwright-core": "cli.js"
},
"engines": {
"node": ">=18"
}
}
}
}

5
package.json Normal file
View File

@@ -0,0 +1,5 @@
{
"devDependencies": {
"playwright": "^1.59.1"
}
}

1
start.sh
View File

@@ -6,6 +6,7 @@ S3_ENDPOINT="${S3_ENDPOINT#http://}"
# Init SQL para o terminal web (credenciais não ficam expostas como env vars)
cat > /app/ssh_init.sql <<SQL
INSTALL httpfs;
LOAD httpfs;
SET s3_endpoint='${S3_ENDPOINT}';
SET s3_access_key_id='${AWS_ACCESS_KEY_ID}';