Update infrastructure and auth configuration, add package management files

.gitignore

@@ -7,5 +7,6 @@ done_transfers.txt
 **/target
 *.log
 .venv
+node_modules/
 .cocoindex_code
 .cocoindex_code

Caddyfile

@@ -17,6 +17,15 @@
         }
 
         handle /query {
+            header Access-Control-Allow-Origin *
+            header Access-Control-Allow-Methods "GET, POST, OPTIONS"
+            header Access-Control-Allow-Headers "X-Password, Content-Type"
+
+            @options method OPTIONS
+            handle @options {
+                respond 204
+            }
+
             reverse_proxy 127.0.0.1:8081
         }
 
@@ -52,6 +61,15 @@
         }
 
         handle /query {
+            header Access-Control-Allow-Origin *
+            header Access-Control-Allow-Methods "GET, POST, OPTIONS"
+            header Access-Control-Allow-Headers "X-Password, Content-Type"
+
+            @options method OPTIONS
+            handle @options {
+                respond 204
+            }
+
             reverse_proxy 127.0.0.1:8081
         }
 

Dockerfile

@@ -13,18 +13,27 @@ RUN apt-get update -qq && \
     rm /tmp/libduckdb.zip && \
     apt-get clean && rm -rf /var/lib/apt/lists/*
 
-WORKDIR /build
+WORKDIR /build/ask
 COPY ask/Cargo.toml ask/Cargo.lock ./
 COPY ask/src ./src
 
 RUN rustup target add x86_64-unknown-linux-musl && \
-    mkdir -p /build/.cargo && \
+    mkdir -p /build/ask/.cargo && \
-    printf '[target.x86_64-unknown-linux-musl]\nlinker = "gcc"\n' > /build/.cargo/config.toml && \
+    printf '[target.x86_64-unknown-linux-musl]\nlinker = "gcc"\n' > /build/ask/.cargo/config.toml && \
-    printf 'rustflags = ["-L", "/usr/local/lib", "-C", "target-feature=+crt-static"]\n' >> /build/.cargo/config.toml && \
+    printf 'rustflags = ["-L", "/usr/local/lib", "-C", "target-feature=+crt-static"]\n' >> /build/ask/.cargo/config.toml && \
     sed -i 's/features = \["bundled"\]/default-features = false/' Cargo.toml && \
     rm -f Cargo.lock && \
     cargo build --release --target x86_64-unknown-linux-musl
 
+WORKDIR /build/dbquery
+COPY dbquery/Cargo.toml dbquery/Cargo.lock* ./
+COPY dbquery/src ./src
+
+RUN mkdir -p /build/dbquery/.cargo && \
+    printf '[target.x86_64-unknown-linux-musl]\nlinker = "gcc"\n' > /build/dbquery/.cargo/config.toml && \
+    printf 'rustflags = ["-C", "target-feature=+crt-static"]\n' >> /build/dbquery/.cargo/config.toml && \
+    cargo build --release --target x86_64-unknown-linux-musl
+
 FROM --platform=linux/amd64 debian:12-slim
 
 RUN apt-get update -qq && \
@@ -54,7 +63,8 @@ RUN apt-get update -qq && \
 
 WORKDIR /app
 
-COPY --from=builder /build/target/x86_64-unknown-linux-musl/release/ask ./ask
+COPY --from=builder /build/ask/target/x86_64-unknown-linux-musl/release/ask ./ask
+COPY --from=builder /build/dbquery/target/x86_64-unknown-linux-musl/release/dbquery ./dbquery
 COPY ask/system_prompt.md ./system_prompt.md
 COPY data/basedosdados.duckdb ./data/
 COPY context ./context/
@@ -62,7 +72,7 @@ COPY auth.py ./
 COPY start.sh ./
 COPY Caddyfile ./
 
-RUN chmod +x start.sh ask
+RUN chmod +x start.sh ask dbquery
 
 EXPOSE 8080
 

auth.py

@@ -1,10 +1,32 @@
 #!/usr/bin/env python3
 """Minimal cookie-session auth gate for DuckDB shell."""
-import hmac, hashlib, os, secrets, subprocess, time
+import hmac, hashlib, json, os, secrets, subprocess, time
 from http.server import HTTPServer, BaseHTTPRequestHandler
+from socketserver import ThreadingMixIn
 from urllib.parse import parse_qs
 
 PASSWORD = os.environ.get('BASIC_AUTH_PASSWORD', '').encode()
+
+_INIT_SQL = None
+
+def _run_query(sql, json_mode=True):
+    global _INIT_SQL
+    if _INIT_SQL is None:
+        with open('/app/ssh_init.sql') as f:
+            _INIT_SQL = f.read()
+    if json_mode:
+        sql = '.mode json\n' + sql
+    try:
+        r = subprocess.run(
+            ['duckdb', '-readonly', '/app/data/basedosdados.duckdb'],
+            input=_INIT_SQL + '\n' + sql, capture_output=True, text=True, timeout=120
+        )
+        if r.stdout.strip().startswith('['):
+            return r.stdout.encode()
+        err = (r.stderr or r.stdout or 'unknown DuckDB error').strip()
+        return json.dumps({'error': err}).encode()
+    except subprocess.TimeoutExpired:
+        return json.dumps({'error': 'query timed out after 120s'}).encode()
 _SECRET  = secrets.token_bytes(32)
 
 def _make_token():
@@ -51,6 +73,18 @@ class H(BaseHTTPRequestHandler):
                 self.send_response(302)
                 self.send_header('Location', '/login')
                 self.end_headers()
+        elif self.path.startswith('/query'):
+            from urllib.parse import urlparse, parse_qs
+            pwd = self.headers.get('X-Password', '').encode()
+            if not hmac.compare_digest(pwd, PASSWORD):
+                self._resp(401, b'Unauthorized\n')
+                return
+            qs = parse_qs(urlparse(self.path).query)
+            sql = qs.get('q', [''])[0]
+            if not sql:
+                self._resp(400, b'missing ?q=\n')
+                return
+            self._resp(200, _run_query(sql), 'application/json')
         else:
             self._resp(200, LOGIN_HTML, 'text/html; charset=utf-8')
 
@@ -61,15 +95,7 @@ class H(BaseHTTPRequestHandler):
                 self._resp(401, b'Unauthorized\n')
                 return
             sql = self.rfile.read(int(self.headers.get('Content-Length', 0))).decode(errors='replace')
-            try:
+            self._resp(200, _run_query(sql), 'application/json')
-                r = subprocess.run(
-                    ['duckdb', '-readonly', '--init', '/app/ssh_init.sql', '/app/basedosdados.duckdb'],
-                    input=sql, capture_output=True, text=True, timeout=120
-                )
-                out = (r.stdout + r.stderr).encode()
-            except subprocess.TimeoutExpired:
-                out = b'timeout\n'
-            self._resp(200, out, 'text/plain; charset=utf-8')
             return
         body = self.rfile.read(int(self.headers.get('Content-Length', 0))).decode(errors='replace')
         pwd  = parse_qs(body).get('password', [''])[0].encode()
@@ -81,6 +107,10 @@ class H(BaseHTTPRequestHandler):
         else:
             self._resp(200, LOGIN_HTML, 'text/html; charset=utf-8')
 
+    def do_OPTIONS(self):
+        self.send_response(204)
+        self.end_headers()
+
     def _resp(self, code, body=b'', ct='text/plain'):
         self.send_response(code)
         if body:
@@ -93,4 +123,7 @@ class H(BaseHTTPRequestHandler):
     def log_message(self, *_):
         pass
 
-HTTPServer(('127.0.0.1', 8081), H).serve_forever()
+class ThreadedHTTPServer(ThreadingMixIn, HTTPServer):
+    daemon_threads = True
+
+ThreadedHTTPServer(('127.0.0.1', 8081), H).serve_forever()

dbquery/Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "dbquery"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+reqwest = { version = "0.12", features = ["blocking", "rustls-tls"], default-features = false }
+clap = { version = "4", features = ["derive", "env"] }

dbquery/src/main.rs

@@ -0,0 +1,68 @@
+use clap::{Parser, ValueEnum};
+use std::io::{self, Read};
+
+#[derive(Clone, ValueEnum)]
+enum Method {
+    Get,
+    Post,
+}
+
+#[derive(Parser)]
+#[command(about = "Query a db endpoint (always returns JSON)")]
+struct Args {
+    /// SQL query (if omitted, reads from stdin)
+    query: Option<String>,
+
+    /// Base server URL
+    #[arg(short, long, default_value = "https://db.xn--2dk.xyz")]
+    server: String,
+
+    /// API password (or set BASIC_AUTH_PASSWORD env var)
+    #[arg(short, long, env = "BASIC_AUTH_PASSWORD")]
+    password: String,
+
+    /// HTTP method: GET sends query as ?q= param, POST sends as body
+    #[arg(short = 'X', long, value_enum, default_value = "post")]
+    method: Method,
+}
+
+fn main() {
+    let args = Args::parse();
+
+    let query = match args.query {
+        Some(q) => q,
+        None => {
+            let mut buf = String::new();
+            io::stdin().read_to_string(&mut buf).expect("failed to read stdin");
+            buf
+        }
+    };
+
+    let base = format!("{}/query", args.server.trim_end_matches('/'));
+    let client = reqwest::blocking::Client::new();
+
+    let resp = match args.method {
+        Method::Get => client
+            .get(&base)
+            .header("X-Password", &args.password)
+            .query(&[("q", &query)])
+            .send()
+            .expect("request failed"),
+        Method::Post => client
+            .post(&base)
+            .header("X-Password", &args.password)
+            .body(query)
+            .send()
+            .expect("request failed"),
+    };
+
+    let status = resp.status();
+    let body = resp.text().expect("failed to read response");
+
+    if !status.is_success() {
+        eprintln!("Error {status}: {body}");
+        std::process::exit(1);
+    }
+
+    print!("{body}");
+}

package-lock.json

@@ -0,0 +1,59 @@
+{
+  "name": "baseldosdados",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {
+    "": {
+      "devDependencies": {
+        "playwright": "^1.59.1"
+      }
+    },
+    "node_modules/fsevents": {
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
+      "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
+      "dev": true,
+      "hasInstallScript": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
+      }
+    },
+    "node_modules/playwright": {
+      "version": "1.59.1",
+      "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.59.1.tgz",
+      "integrity": "sha512-C8oWjPR3F81yljW9o5OxcWzfh6avkVwDD2VYdwIGqTkl+OGFISgypqzfu7dOe4QNLL2aqcWBmI3PMtLIK233lw==",
+      "dev": true,
+      "license": "Apache-2.0",
+      "dependencies": {
+        "playwright-core": "1.59.1"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      },
+      "optionalDependencies": {
+        "fsevents": "2.3.2"
+      }
+    },
+    "node_modules/playwright-core": {
+      "version": "1.59.1",
+      "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.59.1.tgz",
+      "integrity": "sha512-HBV/RJg81z5BiiZ9yPzIiClYV/QMsDCKUyogwH9p3MCP6IYjUFu/MActgYAvK0oWyV9NlwM3GLBjADyWgydVyg==",
+      "dev": true,
+      "license": "Apache-2.0",
+      "bin": {
+        "playwright-core": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    }
+  }
+}

package.json

@@ -0,0 +1,5 @@
+{
+  "devDependencies": {
+    "playwright": "^1.59.1"
+  }
+}

start.sh

@@ -6,6 +6,7 @@ S3_ENDPOINT="${S3_ENDPOINT#http://}"
 
 # Init SQL para o terminal web (credenciais não ficam expostas como env vars)
 cat > /app/ssh_init.sql <<SQL
+INSTALL httpfs;
 LOAD httpfs;
 SET s3_endpoint='${S3_ENDPOINT}';
 SET s3_access_key_id='${AWS_ACCESS_KEY_ID}';