scrapeninja

README

ScrapeNinja Web scraper Go API Client

A slim Go client for interacting with the ScrapeNinja API

More info about ScrapeNinja

• Official website - https://scrapeninja.net
• Rapid API documentation - https://rapidapi.com/restyler/api/scrapeninja

Installation

go get -u github.com/backdevs/scrapeninja-go

Usage

package main

import (
	"encoding/json"
	"fmt"
	"github.com/backdevs/scrapeninja-go"
	"log"
	"net/http"
	"strings"
)

type Post struct {
	ID        string   `json:"id"`
	Title     string   `json:"title"`
	Body      string   `json:"body"`
	UserID    int      `json:"userId"`
	Tags      []string `json:"tags"`
	Reactions int      `json:"reactions"`
}

func main() {
	ninja := scrapeninja.New("Your-API-Key")

	postJson, err := json.Marshal(&Post{
		ID:     "1",
		UserID: 12,
		Title:  "ScrapeNinja Web scraper Go API Client",
		Body:   "Lorem ipsum etc.",
	})
	if err != nil {
		log.Fatal(err)
	}

	response, err := ninja.Scrape(&scrapeninja.ScrapeRequest{
		Url:    "https://dummyjson.com/posts/1",
		Method: http.MethodPut,
		Proxy:  scrapeninja.ProxyEU,
		Data:   string(postJson),
		Headers: map[string]string{
			"Content-Type": "application/json",
			"Accept":       "application/json",
		},
	})
	if err != nil {
		log.Fatal(err)
	}

	var post Post
	err = json.NewDecoder(strings.NewReader(response.Body)).Decode(&post)
	if err != nil {
		log.Fatal(err)
	}

	fmt.Println(post.Title)
}

Documentation

Index

• Constants
• Variables
• type Client
  • func New(apiKey string) Client
  • func (c *Client) Scrape(request *ScrapeRequest) (*ScrapeResponse, error)
• type ScrapeRequest
  • func (r *ScrapeRequest) MarshalJSON() ([]byte, error)
• type ScrapeResponse

Constants

const (
	ProxyDefault string = "default"
	ProxyUS      string = "us"
	ProxyEU      string = "eu"
	ProxyEU4G    string = "4g-eu"
	ProxyDE      string = "de"
	ProxyFR      string = "fr"
	ProxyBR      string = "br"
)

Variables

var (
	BaseUrl                    = "https://scrapeninja.p.rapidapi.com"
	ContentType                = "application/json"
	RapidAPIHost               = "scrapeninja.p.rapidapi.com"
	Timeout      time.Duration = 30
)

Types

type Client

type Client struct {
	// contains filtered or unexported fields
}

func New

func New(apiKey string) Client

func (*Client) Scrape

func (c *Client) Scrape(request *ScrapeRequest) (*ScrapeResponse, error)

type ScrapeRequest

type ScrapeRequest struct {
	Url     string            `json:"url"`
	Method  string            `json:"method"`
	Headers map[string]string `json:"headers,omitempty"`
	Data    string            `json:"body,omitempty"`
	Proxy   string            `json:"geo,omitempty"`
}

func (*ScrapeRequest) MarshalJSON

func (r *ScrapeRequest) MarshalJSON() ([]byte, error)

type ScrapeResponse

type ScrapeResponse struct {
	Info struct {
		Version       string            `json:"version"`
		StatusCode    int               `json:"statusCode"`
		StatusMessage string            `json:"statusMessage"`
		Headers       map[string]string `json:"headers"`
	} `json:"info"`
	Body string `json:"body"`
}