Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 32 additions & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,14 @@ on:
- 'v*'
pull_request:
branches: [ main ]
# Dispatched by unravel-router's bump workflow, which tests the CLI against
# staging before every aperiodic.io release.
workflow_dispatch:
inputs:
base_url:
description: 'API base URL to test against (empty: production)'
required: false
default: ''

jobs:
test:
Expand All @@ -23,9 +31,32 @@ jobs:
uses: actions/setup-go@v6
with:
go-version: '1.24'
# A missing staging secret would fall back to production or fail every
# request at Cloudflare Access; say which one instead.
- name: Require the staging secrets
if: inputs.base_url
shell: bash
env:
APERIODIC_STAGING_API_KEY: ${{ secrets.APERIODIC_STAGING_API_KEY }}
CF_ACCESS_CLIENT_ID: ${{ secrets.CF_ACCESS_CLIENT_ID }}
CF_ACCESS_CLIENT_SECRET: ${{ secrets.CF_ACCESS_CLIENT_SECRET }}
run: |
missing=0
for name in APERIODIC_STAGING_API_KEY CF_ACCESS_CLIENT_ID CF_ACCESS_CLIENT_SECRET; do
if [ -z "${!name}" ]; then
echo "::error::The $name secret is not set"
missing=1
fi
done
exit $missing
- name: Test
env:
APERIODIC_API_KEY: ${{ secrets.APERIODIC_API_KEY }}
# Push and PR runs test production. A dispatch with a base_url tests
# that API (staging) with its key. Either key must be on Prime + Raw.
APERIODIC_API_KEY: ${{ inputs.base_url && secrets.APERIODIC_STAGING_API_KEY || secrets.APERIODIC_API_KEY }}
APERIODIC_API_URL: ${{ inputs.base_url || '' }}
CF_ACCESS_CLIENT_ID: ${{ secrets.CF_ACCESS_CLIENT_ID }}
CF_ACCESS_CLIENT_SECRET: ${{ secrets.CF_ACCESS_CLIENT_SECRET }}
run: go test -v ./...

build:
Expand Down
19 changes: 13 additions & 6 deletions client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package aperiodic

import (
"bytes"
"cmp"
"fmt"
"io"
"net/http"
Expand All @@ -20,12 +21,18 @@ func requireAPIKey(t *testing.T) string {
t.Fatal("APERIODIC_API_KEY environment variable not set")
}

// Force production base URL for all integration tests
t.Setenv("APERIODIC_API_URL", DefaultBaseURL)
useLiveAPI(t)

return apiKey
}

// useLiveAPI points a test at the live API: production, unless
// APERIODIC_API_URL names another (CI sets staging before a release).
func useLiveAPI(t *testing.T) {
t.Helper()
t.Setenv("APERIODIC_API_URL", cmp.Or(os.Getenv("APERIODIC_API_URL"), DefaultBaseURL))
}

func TestHandleAPIError(t *testing.T) {
client := NewAperiodicClient("test-key")

Expand Down Expand Up @@ -208,7 +215,7 @@ func TestCLI_NoPreviewUsesDataPath(t *testing.T) {
// where invalid-key tests run unconditionally and assert specific error codes.

func TestCLI_Symbols_InvalidAPIKey(t *testing.T) {
t.Setenv("APERIODIC_API_URL", DefaultBaseURL)
useLiveAPI(t)
t.Setenv("APERIODIC_API_KEY", "invalid-key")

_, stderr, code := runCLI("symbols", "-exchange", "binance-futures")
Expand All @@ -218,7 +225,7 @@ func TestCLI_Symbols_InvalidAPIKey(t *testing.T) {
}

func TestCLI_Symbols_HyperliquidPerps_InvalidAPIKey(t *testing.T) {
t.Setenv("APERIODIC_API_URL", DefaultBaseURL)
useLiveAPI(t)
t.Setenv("APERIODIC_API_KEY", "invalid-key")

_, stderr, code := runCLI("symbols", "-exchange", "hyperliquid-perps")
Expand All @@ -228,7 +235,7 @@ func TestCLI_Symbols_HyperliquidPerps_InvalidAPIKey(t *testing.T) {
}

func TestCLI_OHLCV_InvalidAPIKey(t *testing.T) {
t.Setenv("APERIODIC_API_URL", DefaultBaseURL)
useLiveAPI(t)
t.Setenv("APERIODIC_API_KEY", "invalid-key")

outputDir := t.TempDir()
Expand All @@ -247,7 +254,7 @@ func TestCLI_OHLCV_InvalidAPIKey(t *testing.T) {
}

func TestCLI_OHLCV_HyperliquidPerps_InvalidAPIKey(t *testing.T) {
t.Setenv("APERIODIC_API_URL", DefaultBaseURL)
useLiveAPI(t)
t.Setenv("APERIODIC_API_KEY", "invalid-key")

outputDir := t.TempDir()
Expand Down
2 changes: 1 addition & 1 deletion processor_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ func TestCLI_OHLCV_HyperliquidPerps_Download(t *testing.T) {
}

func TestCLI_VTWAP_InvalidAPIKey(t *testing.T) {
t.Setenv("APERIODIC_API_URL", DefaultBaseURL)
useLiveAPI(t)
t.Setenv("APERIODIC_API_KEY", "invalid-key")

outputDir := t.TempDir()
Expand Down
151 changes: 151 additions & 0 deletions raw_live_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,151 @@
package aperiodic

// Raw data against the live API (production, or APERIODIC_API_URL): what a
// user running the CLI gets. The files are the June 2025 BTC perpetuals, in
// the raw buckets since the proof of concept. The paid download needs
// APERIODIC_API_KEY to be on the Prime + Raw plan.

import (
"bytes"
"os"
"regexp"
"strings"
"testing"
)

var isoDay = regexp.MustCompile(`^\d{4}-\d{2}-\d{2}$`)

// requireJuneParquet checks the June 2025 file landed where RawFilePath says,
// and is Parquet rather than a storage error page.
func requireJuneParquet(t *testing.T, outputDir string, dataset RawDataset) {
t.Helper()

path, err := RawFilePath(outputDir, dataset, ExchangeBinanceFutures, rawTestSymbol, "2025-06")
if err != nil {
t.Fatal(err)
}
content, err := os.ReadFile(path)
if err != nil {
t.Fatalf("expected the June file at %s: %v", path, err)
}
if !bytes.HasPrefix(content, []byte("PAR1")) || !bytes.HasSuffix(content, []byte("PAR1")) {
t.Fatalf("%s is not a Parquet file (%d bytes)", path, len(content))
}
}

func TestCLI_Raw_Live_CoverageListsEveryDataset(t *testing.T) {
useLiveAPI(t)

stdout, stderr, code := runCLI("raw", "coverage")
if code != 0 {
t.Fatalf("expected exit code 0, got %d; stderr: %s", code, stderr)
}
for _, dataset := range RawDatasets {
if !strings.Contains(stdout, "\n"+string(dataset)+" ") {
t.Errorf("expected a summary row for %s, got:\n%s", dataset, stdout)
}
}
}

func TestCLI_Raw_Live_CoverageOfOneSymbol(t *testing.T) {
useLiveAPI(t)

stdout, stderr, code := runCLI(
"raw", "coverage",
"-dataset", "trades",
"-exchange", "binance-futures",
"-symbol", rawTestSymbol,
)
if code != 0 {
t.Fatalf("expected exit code 0, got %d; stderr: %s", code, stderr)
}

lines := strings.Split(strings.TrimSpace(stdout), "\n")
if len(lines) != 2 {
t.Fatalf("expected a header and one row, got:\n%s", stdout)
}
// DATASET EXCHANGE SYMBOL FIRST LAST DAYS ...
row := strings.Fields(lines[1])
if len(row) < 5 || row[2] != rawTestSymbol {
t.Fatalf("unexpected coverage row: %q", lines[1])
}
first, last := row[3], row[4]
if !isoDay.MatchString(first) || !isoDay.MatchString(last) {
t.Fatalf("expected FIRST and LAST as YYYY-MM-DD, got %q and %q", first, last)
}
if first > "2025-06-01" || last < "2025-06-30" {
t.Errorf("expected the coverage to span June 2025, got %s to %s", first, last)
}
}

func TestCLI_Raw_Live_PreviewNeedsNoKey(t *testing.T) {
useLiveAPI(t)
t.Setenv("APERIODIC_API_KEY", "")

outputDir := t.TempDir()
stdout, stderr, code := runCLI(
"raw", "funding_rate", "-preview",
"-symbol", rawTestSymbol,
"-output-dir", outputDir,
)
if code != 0 {
t.Fatalf("expected exit code 0, got %d; stderr: %s", code, stderr)
}
if !strings.Contains(stdout, "Successfully downloaded 1 files") {
t.Errorf("expected one file downloaded, got: %s", stdout)
}
requireJuneParquet(t, outputDir, RawFundingRate)
}

func TestCLI_Raw_Live_DemoKeyIsRefusedOutsideThePreview(t *testing.T) {
useLiveAPI(t)
t.Setenv("APERIODIC_API_KEY", DemoAPIKey)

outputDir := t.TempDir()
_, stderr, code := runCLI(
"raw", "trades",
"-symbol", rawTestSymbol,
"-start-date", "2025-06-01",
"-end-date", "2025-06-02",
"-output-dir", outputDir,
)
if code != 1 {
t.Fatalf("expected exit code 1, got %d; stderr: %s", code, stderr)
}
if !strings.Contains(stderr, "401") {
t.Errorf("expected a 401, got: %s", stderr)
}
}

// Downloads the whole June trades file (a few hundred MB): the size raw users
// stream, which the small preview file above does not exercise.
func TestCLI_Raw_Live_DownloadsAndThenSkipsTheJuneFile(t *testing.T) {
requireAPIKey(t)

outputDir := t.TempDir()
args := []string{
"raw", "trades",
"-exchange", "binance-futures",
"-symbol", rawTestSymbol,
"-start-date", "2025-06-01",
"-end-date", "2025-06-02",
"-output-dir", outputDir,
}

stdout, stderr, code := runCLI(args...)
if code != 0 {
t.Fatalf("expected exit code 0, got %d; stderr: %s", code, stderr)
}
if !strings.Contains(stdout, "Successfully downloaded 1 files") {
t.Errorf("expected one file downloaded, got: %s", stdout)
}
requireJuneParquet(t, outputDir, RawTrades)

stdout, stderr, code = runCLI(args...)
if code != 0 {
t.Fatalf("expected exit code 0 on the second run, got %d; stderr: %s", code, stderr)
}
if !strings.Contains(stdout, "Skipping 1 already present") {
t.Errorf("expected the second run to skip the file, got: %s", stdout)
}
}
Loading