feat: add Polymarket and Kalshi prediction market extractors
Add site extractors for two prediction market platforms using their public REST APIs (no browser/Playwright dependency needed). Polymarket: search via Gamma API, get events/markets by slug/ID/URL. Handles inconsistent JSON encoding (stringified arrays). Kalshi: get markets/events by ticker/URL, search via client-side filtering of open events (no text search API available). Both include CLI tools and comprehensive test suites using httptest. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
326
sites/polymarket/api.go
Normal file
326
sites/polymarket/api.go
Normal file
@@ -0,0 +1,326 @@
|
||||
package polymarket
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultGammaBaseURL = "https://gamma-api.polymarket.com"
|
||||
)
|
||||
|
||||
// stringOrArray handles Polymarket's inconsistent JSON encoding where fields
|
||||
// can be either a JSON array or a stringified JSON array (e.g. "[\"Yes\",\"No\"]").
|
||||
type stringOrArray []string
|
||||
|
||||
func (s *stringOrArray) UnmarshalJSON(data []byte) error {
|
||||
// Try as regular array first
|
||||
var arr []string
|
||||
if err := json.Unmarshal(data, &arr); err == nil {
|
||||
*s = arr
|
||||
return nil
|
||||
}
|
||||
|
||||
// Try as stringified array
|
||||
var str string
|
||||
if err := json.Unmarshal(data, &str); err == nil {
|
||||
if err := json.Unmarshal([]byte(str), &arr); err == nil {
|
||||
*s = arr
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
*s = nil
|
||||
return nil
|
||||
}
|
||||
|
||||
// floatOrArray handles the same pattern for numeric arrays.
|
||||
type floatOrArray []float64
|
||||
|
||||
func (f *floatOrArray) UnmarshalJSON(data []byte) error {
|
||||
// Try as regular array first
|
||||
var arr []float64
|
||||
if err := json.Unmarshal(data, &arr); err == nil {
|
||||
*f = arr
|
||||
return nil
|
||||
}
|
||||
|
||||
// Try as array of strings
|
||||
var strArr []string
|
||||
if err := json.Unmarshal(data, &strArr); err == nil {
|
||||
result := make([]float64, 0, len(strArr))
|
||||
for _, s := range strArr {
|
||||
v, err := strconv.ParseFloat(s, 64)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parsing price %q: %w", s, err)
|
||||
}
|
||||
result = append(result, v)
|
||||
}
|
||||
*f = result
|
||||
return nil
|
||||
}
|
||||
|
||||
// Try as stringified array of strings
|
||||
var str string
|
||||
if err := json.Unmarshal(data, &str); err == nil {
|
||||
if err := json.Unmarshal([]byte(str), &strArr); err == nil {
|
||||
result := make([]float64, 0, len(strArr))
|
||||
for _, s := range strArr {
|
||||
v, err := strconv.ParseFloat(s, 64)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parsing price %q: %w", s, err)
|
||||
}
|
||||
result = append(result, v)
|
||||
}
|
||||
*f = result
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
*f = nil
|
||||
return nil
|
||||
}
|
||||
|
||||
// apiMarket is the JSON shape returned by the Gamma API for a market.
|
||||
type apiMarket struct {
|
||||
ID string `json:"id"`
|
||||
Question string `json:"question"`
|
||||
Slug string `json:"slug"`
|
||||
Outcomes stringOrArray `json:"outcomes"`
|
||||
OutcomePrices floatOrArray `json:"outcomePrices"`
|
||||
Volume flexFloat `json:"volume"`
|
||||
Volume24hr float64 `json:"volume24hr"`
|
||||
LiquidityClob float64 `json:"liquidityClob"`
|
||||
EndDate string `json:"endDate"`
|
||||
Active bool `json:"active"`
|
||||
Closed bool `json:"closed"`
|
||||
BestBid float64 `json:"bestBid"`
|
||||
BestAsk float64 `json:"bestAsk"`
|
||||
LastTradePrice float64 `json:"lastTradePrice"`
|
||||
Description string `json:"description"`
|
||||
GroupItemTitle string `json:"groupItemTitle"`
|
||||
}
|
||||
|
||||
// flexFloat handles volume fields that are sometimes strings, sometimes numbers.
|
||||
type flexFloat float64
|
||||
|
||||
func (f *flexFloat) UnmarshalJSON(data []byte) error {
|
||||
var num float64
|
||||
if err := json.Unmarshal(data, &num); err == nil {
|
||||
*f = flexFloat(num)
|
||||
return nil
|
||||
}
|
||||
|
||||
var str string
|
||||
if err := json.Unmarshal(data, &str); err == nil {
|
||||
v, err := strconv.ParseFloat(str, 64)
|
||||
if err != nil {
|
||||
*f = 0
|
||||
return nil
|
||||
}
|
||||
*f = flexFloat(v)
|
||||
return nil
|
||||
}
|
||||
|
||||
*f = 0
|
||||
return nil
|
||||
}
|
||||
|
||||
// apiEvent is the JSON shape returned by the Gamma API for an event.
|
||||
type apiEvent struct {
|
||||
ID string `json:"id"`
|
||||
Title string `json:"title"`
|
||||
Slug string `json:"slug"`
|
||||
Description string `json:"description"`
|
||||
Markets []apiMarket `json:"markets"`
|
||||
Volume float64 `json:"volume"`
|
||||
Volume24hr float64 `json:"volume24hr"`
|
||||
Liquidity float64 `json:"liquidity"`
|
||||
Active bool `json:"active"`
|
||||
Closed bool `json:"closed"`
|
||||
}
|
||||
|
||||
// apiSearchResponse is the JSON shape returned by the public-search endpoint.
|
||||
type apiSearchResponse struct {
|
||||
Events []apiEvent `json:"events"`
|
||||
}
|
||||
|
||||
func apiMarketToMarket(m apiMarket) Market {
|
||||
return Market{
|
||||
ID: m.ID,
|
||||
Question: m.Question,
|
||||
Slug: m.Slug,
|
||||
Outcomes: []string(m.Outcomes),
|
||||
OutcomePrices: []float64(m.OutcomePrices),
|
||||
Volume: float64(m.Volume),
|
||||
Volume24h: m.Volume24hr,
|
||||
Liquidity: m.LiquidityClob,
|
||||
EndDate: m.EndDate,
|
||||
Active: m.Active,
|
||||
Closed: m.Closed,
|
||||
BestBid: m.BestBid,
|
||||
BestAsk: m.BestAsk,
|
||||
LastPrice: m.LastTradePrice,
|
||||
Description: m.Description,
|
||||
}
|
||||
}
|
||||
|
||||
func apiEventToEvent(e apiEvent) Event {
|
||||
markets := make([]Market, 0, len(e.Markets))
|
||||
for _, m := range e.Markets {
|
||||
markets = append(markets, apiMarketToMarket(m))
|
||||
}
|
||||
|
||||
return Event{
|
||||
ID: e.ID,
|
||||
Title: e.Title,
|
||||
Slug: e.Slug,
|
||||
Description: e.Description,
|
||||
Markets: markets,
|
||||
Volume: e.Volume,
|
||||
Volume24h: e.Volume24hr,
|
||||
Liquidity: e.Liquidity,
|
||||
Active: e.Active,
|
||||
Closed: e.Closed,
|
||||
}
|
||||
}
|
||||
|
||||
func (c Config) doGet(ctx context.Context, endpoint string) ([]byte, error) {
|
||||
base := c.BaseURL
|
||||
if base == "" {
|
||||
base = defaultGammaBaseURL
|
||||
}
|
||||
|
||||
reqURL := base + endpoint
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, reqURL, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("creating request: %w", err)
|
||||
}
|
||||
req.Header.Set("Accept", "application/json")
|
||||
|
||||
client := c.HTTPClient
|
||||
if client == nil {
|
||||
client = http.DefaultClient
|
||||
}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("executing request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading response: %w", err)
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("API returned status %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
return body, nil
|
||||
}
|
||||
|
||||
func (c Config) searchAPI(ctx context.Context, query string) ([]Event, error) {
|
||||
endpoint := "/public-search?q=" + url.QueryEscape(query)
|
||||
|
||||
body, err := c.doGet(ctx, endpoint)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("searching: %w", err)
|
||||
}
|
||||
|
||||
var resp apiSearchResponse
|
||||
if err := json.Unmarshal(body, &resp); err != nil {
|
||||
return nil, fmt.Errorf("parsing search response: %w", err)
|
||||
}
|
||||
|
||||
events := make([]Event, 0, len(resp.Events))
|
||||
for _, e := range resp.Events {
|
||||
events = append(events, apiEventToEvent(e))
|
||||
}
|
||||
|
||||
return events, nil
|
||||
}
|
||||
|
||||
func (c Config) getEventBySlug(ctx context.Context, slug string) (*Event, error) {
|
||||
body, err := c.doGet(ctx, "/events/slug/"+url.PathEscape(slug))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("getting event: %w", err)
|
||||
}
|
||||
|
||||
var raw apiEvent
|
||||
if err := json.Unmarshal(body, &raw); err != nil {
|
||||
return nil, fmt.Errorf("parsing event: %w", err)
|
||||
}
|
||||
|
||||
ev := apiEventToEvent(raw)
|
||||
return &ev, nil
|
||||
}
|
||||
|
||||
func (c Config) getEventByID(ctx context.Context, id string) (*Event, error) {
|
||||
body, err := c.doGet(ctx, "/events/"+url.PathEscape(id))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("getting event: %w", err)
|
||||
}
|
||||
|
||||
var raw apiEvent
|
||||
if err := json.Unmarshal(body, &raw); err != nil {
|
||||
return nil, fmt.Errorf("parsing event: %w", err)
|
||||
}
|
||||
|
||||
ev := apiEventToEvent(raw)
|
||||
return &ev, nil
|
||||
}
|
||||
|
||||
func (c Config) getMarketBySlug(ctx context.Context, slug string) (*Market, error) {
|
||||
body, err := c.doGet(ctx, "/markets/slug/"+url.PathEscape(slug))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("getting market: %w", err)
|
||||
}
|
||||
|
||||
var raw apiMarket
|
||||
if err := json.Unmarshal(body, &raw); err != nil {
|
||||
return nil, fmt.Errorf("parsing market: %w", err)
|
||||
}
|
||||
|
||||
m := apiMarketToMarket(raw)
|
||||
return &m, nil
|
||||
}
|
||||
|
||||
// parsePolymarketURL extracts the type ("event" or "market") and slug from a Polymarket URL.
|
||||
// Supported formats:
|
||||
// - https://polymarket.com/event/<slug>
|
||||
// - https://polymarket.com/event/<event-slug>/<market-slug>
|
||||
func parsePolymarketURL(rawURL string) (urlType, slug, marketSlug string, err error) {
|
||||
u, err := url.Parse(rawURL)
|
||||
if err != nil {
|
||||
return "", "", "", fmt.Errorf("parsing URL: %w", err)
|
||||
}
|
||||
|
||||
host := strings.TrimPrefix(u.Host, "www.")
|
||||
if host != "polymarket.com" {
|
||||
return "", "", "", fmt.Errorf("not a Polymarket URL: %s", rawURL)
|
||||
}
|
||||
|
||||
parts := strings.Split(strings.Trim(u.Path, "/"), "/")
|
||||
if len(parts) < 2 {
|
||||
return "", "", "", fmt.Errorf("invalid Polymarket URL path: %s", u.Path)
|
||||
}
|
||||
|
||||
switch parts[0] {
|
||||
case "event":
|
||||
if len(parts) >= 3 {
|
||||
return "event", parts[1], parts[2], nil
|
||||
}
|
||||
return "event", parts[1], "", nil
|
||||
default:
|
||||
return "", "", "", fmt.Errorf("unsupported Polymarket URL type: %s", parts[0])
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user