63 lines
2.0 KiB
Python
63 lines
2.0 KiB
Python
import os
|
|
import duckdb
|
|
import boto3
|
|
from dotenv import load_dotenv
|
|
|
|
load_dotenv()
|
|
|
|
S3_ENDPOINT = os.environ["HETZNER_S3_ENDPOINT"] # https://hel1.your-objectstorage.com
|
|
S3_BUCKET = os.environ["HETZNER_S3_BUCKET"] # baseldosdados
|
|
ACCESS_KEY = os.environ["AWS_ACCESS_KEY_ID"]
|
|
SECRET_KEY = os.environ["AWS_SECRET_ACCESS_KEY"]
|
|
|
|
# Strip protocol for DuckDB httpfs (expects bare hostname)
|
|
s3_host = S3_ENDPOINT.removeprefix("https://").removeprefix("http://")
|
|
|
|
con = duckdb.connect('basedosdados.duckdb')
|
|
|
|
con.execute("INSTALL httpfs; LOAD httpfs;")
|
|
con.execute(f"""
|
|
CREATE OR REPLACE PERSISTENT SECRET hetzner (
|
|
TYPE S3,
|
|
KEY_ID '{ACCESS_KEY}',
|
|
SECRET '{SECRET_KEY}',
|
|
ENDPOINT '{s3_host}',
|
|
URL_STYLE 'path'
|
|
);
|
|
""")
|
|
|
|
# List all dataset/table prefixes in the bucket
|
|
s3 = boto3.client(
|
|
's3',
|
|
endpoint_url=S3_ENDPOINT,
|
|
aws_access_key_id=ACCESS_KEY,
|
|
aws_secret_access_key=SECRET_KEY,
|
|
)
|
|
paginator = s3.get_paginator('list_objects_v2')
|
|
|
|
datasets = {}
|
|
for page in paginator.paginate(Bucket=S3_BUCKET, Delimiter='/'):
|
|
for prefix in page.get('CommonPrefixes', []):
|
|
dataset = prefix['Prefix'].rstrip('/')
|
|
datasets[dataset] = []
|
|
for page2 in paginator.paginate(Bucket=S3_BUCKET,
|
|
Prefix=dataset + '/',
|
|
Delimiter='/'):
|
|
for p in page2.get('CommonPrefixes', []):
|
|
table = p['Prefix'].rstrip('/').split('/')[-1]
|
|
datasets[dataset].append(table)
|
|
|
|
# Create schemas and views
|
|
for dataset, tables in datasets.items():
|
|
con.execute(f"CREATE SCHEMA IF NOT EXISTS {dataset}")
|
|
for table in tables:
|
|
path = f"s3://{S3_BUCKET}/{dataset}/{table}/*.parquet"
|
|
con.execute(f"""
|
|
CREATE OR REPLACE VIEW {dataset}.{table} AS
|
|
SELECT * FROM '{path}'
|
|
""")
|
|
print(f"✓ {dataset}.{table}")
|
|
|
|
con.close()
|
|
print("Done! Open with: duckdb --ui basedosdados.duckdb")
|