package scraper import ( "fmt" "io" "log" "net/http" "os" "strconv" "strings" "time" "golang.org/x/net/html" ) type Slot struct { Role string // "tank", "healer", "dps" Job string // "whm", "vpr", etc. or "" if not filled Jobs []string // all job abbreviations from title Filled bool } type Party struct { ID string HostFirstName string HostLastName string HostServer string HostName string // "FirstName LastName" for matching Duty string Description string MinILvl int Slots []Slot URL string } // FetchParties loads parties from xivpf.com. // If the environment variable DEBUG_FILE is set to a path, it reads from that file instead. func FetchParties() ([]Party, error) { debugFile := os.Getenv("DEBUG_FILE") if debugFile != "" { log.Printf("[Scraper] DEBUG mode: reading from %s", debugFile) body, err := os.ReadFile(debugFile) if err != nil { return nil, fmt.Errorf("reading debug file: %w", err) } log.Printf("[Scraper] Read %d bytes from debug file", len(body)) parties, err := parseHTML(body) if err != nil { return nil, err } log.Printf("[Scraper] Parsed %d parties from debug file", len(parties)) return parties, nil } client := &http.Client{ Timeout: 30 * time.Second, } log.Println("[Scraper] Fetching https://xivpf.com/listings") req, err := http.NewRequest("GET", "https://xivpf.com/listings", nil) if err != nil { return nil, fmt.Errorf("creating request: %w", err) } req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36") resp, err := client.Do(req) if err != nil { return nil, fmt.Errorf("fetching listings: %w", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode) } body, err := io.ReadAll(resp.Body) if err != nil { return nil, fmt.Errorf("reading response body: %w", err) } log.Printf("[Scraper] Downloaded %d bytes", len(body)) parties, err := parseHTML(body) if err != nil { return nil, err } log.Printf("[Scraper] Parsed %d parties total", len(parties)) return parties, nil } func parseHTML(body []byte) ([]Party, error) { doc, err := html.Parse(strings.NewReader(string(body))) if err != nil { return nil, fmt.Errorf("parsing HTML: %w", err) } listings := findNodes(doc, "div", "listing") log.Printf("[Scraper] Found %d listing divs", len(listings)) var parties []Party for _, node := range listings { party := parseListing(node) if party.HostName != "" { parties = append(parties, party) } } return parties, nil } func parseListing(n *html.Node) Party { var party Party for _, attr := range n.Attr { if attr.Key == "data-id" { party.ID = attr.Val party.URL = fmt.Sprintf("https://xivpf.com/listings/%s", attr.Val) } } // Parse creator: .meta .creator .text creatorNodes := findNodes(n, "div", "creator") for _, cn := range creatorNodes { textNodes := findNodes(cn, "span", "text") if len(textNodes) > 0 { party.parseCreator(textNodes[0]) } } // Parse duty: .duty dutyNodes := findNodes(n, "div", "duty") if len(dutyNodes) > 0 { party.Duty = extractText(dutyNodes[0]) } // Parse description: .description descNodes := findNodes(n, "div", "description") if len(descNodes) > 0 { party.Description = extractText(descNodes[0]) } // Parse MinILvl minILvlNodes := findNodes(n, "div", "middle") if len(minILvlNodes) > 0 { minILvlValueNodes := findNodes(minILvlNodes[0], "div", "value") if len(minILvlValueNodes) > 0 { minILvlText := extractText(minILvlValueNodes[0]) minILvlInt, _ := strconv.ParseInt(minILvlText, 10, 64) party.MinILvl = int(minILvlInt) } } // Parse slots: .party > .slot partyNodes := findNodes(n, "div", "party") for _, pn := range partyNodes { slotNodes := findNodes(pn, "div", "slot") for _, sn := range slotNodes { party.Slots = append(party.Slots, parseSlot(sn)) } } return party } func (p *Party) parseCreator(node *html.Node) { text := strings.TrimSpace(extractText(node)) parts := strings.Split(text, "@") if len(parts) != 2 { return } namePart := strings.TrimSpace(parts[0]) serverPart := strings.TrimSpace(parts[1]) nameFields := strings.Fields(namePart) if len(nameFields) < 2 { return } p.HostFirstName = nameFields[0] p.HostLastName = nameFields[1] p.HostServer = serverPart p.HostName = nameFields[0] + " " + nameFields[1] } func parseSlot(n *html.Node) Slot { var slot Slot classes := getClassAttr(n) slot.Filled = containsClass(classes, "filled") for _, c := range classes { switch c { case "tank", "healer", "dps": slot.Role = c } } title := getAttr(n, "title") if title != "" { jobs := strings.Fields(title) if len(jobs) > 0 { slot.Job = strings.ToLower(jobs[0]) slot.Jobs = make([]string, len(jobs)) for i, j := range jobs { slot.Jobs[i] = strings.ToLower(j) } // Only infer role from job if the class already has a role-like indicator // (not "empty" which means any role accepted) if slot.Role == "" && !containsClass(classes, "empty") { slot.Role = inferRoleFromJob(strings.ToUpper(jobs[0])) } } } return slot } var tankJobs = map[string]bool{ "GLA": true, "PGL": true, "MRD": true, "LNC": true, "ARC": true, "CNJ": true, "THM": true, "PLD": true, "WAR": true, "DRK": true, "GNB": true, } var healerJobs = map[string]bool{ "WHM": true, "SCH": true, "AST": true, "SGE": true, } func inferRoleFromJob(job string) string { if tankJobs[job] { return "tank" } if healerJobs[job] { return "healer" } return "dps" } func FormatSlot(s Slot, emojiIDs map[string]int64, roleEmojiMap map[string][]string) string { emoji := func(name string) string { id, ok := emojiIDs[name] if ok { return fmt.Sprintf("<:%s:%d>", name, id) } return "" } if s.Filled { return emoji(s.Job) } // For open slots, check if all jobs belong to one role if len(s.Jobs) > 1 { for role, jobs := range roleEmojiMap { jobSet := make(map[string]bool, len(jobs)) for _, j := range jobs { jobSet[j] = true } allMatch := true for _, j := range s.Jobs { if !jobSet[j] { allMatch = false break } } if allMatch { return emoji(role) } } } // Fallback: use role from class or any if s.Role != "" { return emoji(s.Role) } return emoji("any") } // findNodes finds all elements of a given tag with a specific CSS class. func findNodes(root *html.Node, tag string, className string) []*html.Node { var results []*html.Node var walk func(*html.Node) walk = func(n *html.Node) { if n.Type == html.ElementNode { if n.Data == tag && containsClass(getClassAttr(n), className) { results = append(results, n) } } for c := n.FirstChild; c != nil; c = c.NextSibling { walk(c) } } walk(root) return results } func getClassAttr(n *html.Node) []string { for _, attr := range n.Attr { if attr.Key == "class" { return strings.Fields(attr.Val) } } return nil } func containsClass(classes []string, c string) bool { for _, v := range classes { if v == c { return true } } return false } func getAttr(n *html.Node, key string) string { for _, attr := range n.Attr { if attr.Key == key { return attr.Val } } return "" } // extractText recursively extracts all text content from a node, ignoring tags. func extractText(n *html.Node) string { var sb strings.Builder var walk func(*html.Node) walk = func(node *html.Node) { if node.Type == html.TextNode { sb.WriteString(node.Data) } for c := node.FirstChild; c != nil; c = c.NextSibling { walk(c) } } walk(n) return strings.TrimSpace(sb.String()) }