Initial SDK

go.mod

@@ -0,0 +1,3 @@
+module github.com/mslmio/oxylabs-sdk-go
+
+go 1.21.0

oxylabs/common.go

@@ -0,0 +1,40 @@
+package oxylabs
+
+import (
+	"fmt"
+	"net/url"
+	"strings"
+)
+
+func ValidateURL(
+	inputURL string,
+	host string,
+) error {
+	// Check if url is empty.
+	if inputURL == "" {
+		return fmt.Errorf("url parameter is empty")
+	}
+
+	// Parse the URL
+	parsedURL, err := url.ParseRequestURI(inputURL)
+	if err != nil {
+		return fmt.Errorf("failed to parse URL: %v", err)
+	}
+
+	// Check if the scheme (protocol) is present and non-empty.
+	if parsedURL.Scheme == "" {
+		return fmt.Errorf("url is missing scheme")
+	}
+
+	// Check if the Host is present and non-empty.
+	if parsedURL.Host == "" {
+		return fmt.Errorf("url is missing a host")
+	}
+
+	// Check if the Host matches the expected domain/host.
+	if !strings.Contains(parsedURL.Host, host) {
+		return fmt.Errorf("url does not belong to %s", host)
+	}
+
+	return nil
+}

oxylabs/types.go

@@ -0,0 +1,84 @@
+package oxylabs
+
+type UserAgent string
+
+var (
+	UA_DESKTOP         UserAgent = "desktop"
+	UA_DESKTOP_CHROME  UserAgent = "desktop_chrome"
+	UA_DESKTOP_EDGE    UserAgent = "desktop_edge"
+	UA_DESKTOP_FIREFOX UserAgent = "desktop_firefox"
+	UA_DESKTOP_OPERA   UserAgent = "desktop_opera"
+	UA_DESKTOP_SAFARI  UserAgent = "desktop_safari"
+	UA_MOBILE          UserAgent = "mobile"
+	UA_MOBILE_ANDROID  UserAgent = "mobile_android"
+	UA_MOBILE_IOS      UserAgent = "mobile_ios"
+	UA_TABLET          UserAgent = "tablet"
+	UA_TABLET_ANDROID  UserAgent = "tablet_android"
+	UA_TABLET_IOS      UserAgent = "tablet_ios"
+)
+
+func IsUserAgentValid(ua UserAgent) bool {
+	switch ua {
+	case
+		UA_DESKTOP,
+		UA_DESKTOP_CHROME,
+		UA_DESKTOP_EDGE,
+		UA_DESKTOP_FIREFOX,
+		UA_DESKTOP_OPERA,
+		UA_DESKTOP_SAFARI,
+		UA_MOBILE,
+		UA_MOBILE_ANDROID,
+		UA_MOBILE_IOS,
+		UA_TABLET,
+		UA_TABLET_ANDROID,
+		UA_TABLET_IOS:
+		return true
+	default:
+		return false
+	}
+}
+
+type Render string
+
+var (
+	HTML Render = "html"
+	PNG  Render = "png"
+)
+
+func IsRenderValid(render Render) bool {
+	switch render {
+	case
+		HTML,
+		PNG:
+		return true
+	default:
+		return false
+	}
+}
+
+type Domain string
+
+var (
+	DOMAIN_COM Domain = "com"
+	DOMAIN_RU  Domain = "ru"
+	DOMAIN_UA  Domain = "ua"
+	DOMAIN_BY  Domain = "by"
+	DOMAIN_KZ  Domain = "kz"
+	DOMAIN_TR  Domain = "tr"
+	DOMAIN_CN  Domain = "cn"
+)
+
+type Locale string
+
+var (
+	LOCALE_EN Locale = "en"
+	LOCALE_RU Locale = "ru"
+	LOCALE_BY Locale = "by"
+	LOCALE_DE Locale = "de"
+	LOCALE_FR Locale = "fr"
+	LOCALE_ID Locale = "id"
+	LOCALE_KK Locale = "kk"
+	LOCALE_TT Locale = "tt"
+	LOCALE_TR Locale = "tr"
+	LOCALE_UK Locale = "uk"
+)

oxylabs/utils.go

@@ -0,0 +1,11 @@
+package oxylabs
+
+// Checks if the parameter is in the list of accepted parameters.
+func InList[T comparable](val T, list []T) bool {
+	for _, item := range list {
+		if item == val {
+			return true
+		}
+	}
+	return false
+}

serp/baidu.go

@@ -0,0 +1,143 @@
+package serp
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/mslmio/oxylabs-sdk-go/oxylabs"
+)
+
+// Accepted parameters for baidu.
+var BaiduSearchAcceptedDomainParameters = []oxylabs.Domain{
+	oxylabs.DOMAIN_COM,
+	oxylabs.DOMAIN_CN,
+}
+
+// checkParameterValidity checks validity of baidu search parameters.
+func (opt *BaiduSearchOpts) checkParameterValidity() error {
+	if !oxylabs.InList(opt.Domain, BaiduSearchAcceptedDomainParameters) {
+		return fmt.Errorf("invalid domain parameter: %s", opt.Domain)
+	}
+
+	if !oxylabs.IsUserAgentValid(opt.UserAgent) {
+		return fmt.Errorf("invalid user agent parameter: %v", opt.UserAgent)
+	}
+
+	return nil
+}
+
+// checkParameterValidity checks validity of baidu url parameters.
+func (opt *BaiduUrlOpts) checkParameterValidity() error {
+	if !oxylabs.IsUserAgentValid(opt.UserAgent) {
+		return fmt.Errorf("invalid user agent parameter: %v", opt.UserAgent)
+	}
+
+	return nil
+}
+
+type BaiduSearchOpts struct {
+	Domain      oxylabs.Domain
+	StartPage   int
+	Pages       int
+	Limit       int
+	UserAgent   oxylabs.UserAgent
+	CallbackUrl string
+}
+
+// ScrapeBaiduSearch scrapes baidu with baidu_search as source.
+func (c *SerpClient) ScrapeBaiduSearch(
+	query string,
+	opts ...*BaiduSearchOpts,
+) (*Response, error) {
+	// Prepare options
+	opt := &BaiduSearchOpts{}
+	if len(opts) > 0 && opts[len(opts)-1] != nil {
+		opt = opts[len(opts)-1]
+	}
+
+	// Set defaults.
+	SetDefaultDomain(&opt.Domain)
+	SetDefaultStartPage(&opt.StartPage)
+	SetDefaultLimit(&opt.Limit)
+	SetDefaultUserAgent(&opt.UserAgent)
+
+	// Check validity of parameters.
+	err := opt.checkParameterValidity()
+	if err != nil {
+		return nil, err
+	}
+
+	// Prepare payload.
+	payload := map[string]interface{}{
+		"source":          "baidu_search",
+		"domain":          opt.Domain,
+		"query":           query,
+		"start_page":      opt.StartPage,
+		"pages":           opt.Pages,
+		"limit":           opt.Limit,
+		"user_agent_type": opt.UserAgent,
+		"callback_url":    opt.CallbackUrl,
+	}
+	jsonPayload, err := json.Marshal(payload)
+	if err != nil {
+		return nil, fmt.Errorf("error marshalling payload: %v", err)
+	}
+
+	res, err := c.Req(jsonPayload, false, "POST")
+	if err != nil {
+		return nil, err
+	}
+
+	return res, nil
+}
+
+type BaiduUrlOpts struct {
+	UserAgent   oxylabs.UserAgent
+	CallbackUrl string
+}
+
+// ScrapeBaiduUrl scrapes baidu with baidu as source.
+func (c *SerpClient) ScrapeBaiduUrl(
+	url string,
+	opts ...*BaiduUrlOpts,
+) (*Response, error) {
+	// Check validity of url.
+	err := oxylabs.ValidateURL(url, "baidu")
+	if err != nil {
+		return nil, err
+	}
+
+	// Prepare options
+	opt := &BaiduUrlOpts{}
+	if len(opts) > 0 && opts[len(opts)-1] != nil {
+		opt = opts[len(opts)-1]
+	}
+
+	// Set defaults.
+	SetDefaultUserAgent(&opt.UserAgent)
+
+	// Check validity of parameters.
+	err = opt.checkParameterValidity()
+	if err != nil {
+		return nil, err
+	}
+
+	// Prepare payload.
+	payload := map[string]interface{}{
+		"source":          "baidu",
+		"url":             url,
+		"user_agent_type": opt.UserAgent,
+		"callback_url":    opt.CallbackUrl,
+	}
+	jsonPayload, err := json.Marshal(payload)
+	if err != nil {
+		return nil, fmt.Errorf("error marshalling payload: %v", err)
+	}
+
+	res, err := c.Req(jsonPayload, false, "POST")
+	if err != nil {
+		return nil, err
+	}
+
+	return res, nil
+}