Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Initial SDK #2

Merged
merged 27 commits into from
Jan 3, 2024
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
6bb1b50
init commit
maaz-munir Nov 23, 2023
62e861a
some improvements + added bing and baidu
maaz-munir Nov 24, 2023
7ada257
Apply suggestions from code review
maaz-munir Nov 28, 2023
37300b9
added google_search + adjusted or multiple return types
maaz-munir Dec 1, 2023
6e66508
added google source and check for empty url
maaz-munir Dec 1, 2023
fb3c840
Apply suggestions from code review v2
maaz-munir Dec 7, 2023
e72bf48
added remaining google serp sources
maaz-munir Dec 7, 2023
2ffd8e4
comments + some more checks
maaz-munir Dec 8, 2023
f2f2749
check for async runtime models
maaz-munir Dec 11, 2023
14e3d24
Apply suggestions from code review v3 + yandex
maaz-munir Dec 12, 2023
f7dd301
bing and baidu async models + some improvements
maaz-munir Dec 12, 2023
905ed7c
2 google funcs + better error handling with channels
maaz-munir Dec 12, 2023
1420df5
rest of google sources for async polling model
maaz-munir Dec 12, 2023
d14237b
parse checks in google_async + some comment fixes
maaz-munir Dec 16, 2023
d3c3c64
proxy endpoint integration method
maaz-munir Dec 16, 2023
ace829f
send custom headers with proxy endpoint
maaz-munir Dec 16, 2023
0d54a22
make GeoLocation param a ptr
maaz-munir Dec 18, 2023
1a9a06b
refactor async functions
maaz-munir Dec 18, 2023
179164d
update creating payload in google_search funcs
maaz-munir Dec 19, 2023
ce1b8ce
update public func comments
maaz-munir Dec 19, 2023
8fda0b6
Apply suggestions from code review v4
maaz-munir Dec 20, 2023
608084f
update readme
maaz-munir Dec 20, 2023
c689f01
comment
maaz-munir Dec 20, 2023
9270fd9
comments + spelling fixes
maaz-munir Dec 20, 2023
8a4b9c8
update readme
maaz-munir Dec 20, 2023
fe7dda5
update readme
maaz-munir Dec 21, 2023
c250f36
fmt
maaz-munir Dec 21, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 34 additions & 0 deletions proxy/client.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
package proxy

import (
"crypto/tls"
"fmt"
"net/http"
"net/url"
)

// Init for Proxy runtime model.
func Init(
username string,
password string,
) (*http.Client, error) {
// Prepare proxy url.
proxyUrl, err := url.Parse(
fmt.Sprintf(
"http://%s:%s@realtime.oxylabs.io:60000",
username,
password,
),
)
if err != nil {
return nil, fmt.Errorf("error parsing proxy url: %v", err)
}

// Prepare custom transport.
customTransport := &http.Transport{Proxy: http.ProxyURL(proxyUrl)}
customTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}

client := &http.Client{Transport: customTransport}

return client, nil
}
26 changes: 26 additions & 0 deletions proxy/proxy_helpers.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
package proxy

import (
"net/http"

"github.com/mslmio/oxylabs-sdk-go/oxylabs"
)

// Helper functions to add headers to request object.s

func AddGeoLocationHeader(request *http.Request, geoLocation string) {
request.Header.Add("x-oxylabs-geo-location", geoLocation)
}

func AddUserAgentHeader(request *http.Request, userAgent oxylabs.UserAgent) {
request.Header.Add("x-oxylabs-user-agent-type", string(userAgent))
}

func AddRenderHeader(request *http.Request, render oxylabs.Render) {
request.Header.Add("x-oxylabs-render", string(render))
}

func AddParseHeader(request *http.Request, parser string) {
request.Header.Add("x-oxylabs-parse", "1")
request.Header.Add("x-oxylabs-parser", parser)
}
3 changes: 2 additions & 1 deletion serp/async_helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,8 @@ func (c *SerpClientAsync) GetResponse(
}

// PollJobStatus polls the job status and handles the response/error channels.
func (c *SerpClientAsync) PollJobStatus(jobID string,
func (c *SerpClientAsync) PollJobStatus(
jobID string,
parse bool,
responseChan chan *Response,
errChan chan error,
Expand Down
46 changes: 0 additions & 46 deletions serp/client.go
Original file line number Diff line number Diff line change
@@ -1,10 +1,7 @@
package serp

import (
"crypto/tls"
"fmt"
"net/http"
"net/url"
)

type ApiCredentials struct {
Expand Down Expand Up @@ -53,46 +50,3 @@ func InitAsync(
BaseUrl: "https://data.oxylabs.io/v1/queries",
}
}

type SerpClientProxy struct {
HttpClient *http.Client
ApiCredentials *ApiCredentials
}

// Init for Proxy runtime model.
func InitProxy(
username string,
password string,
) *SerpClientProxy {
// Prepare API credentials.
apiCredentials := &ApiCredentials{
Username: username,
Password: password,
}

// Prepare proxy url.
proxyUrl, err := url.Parse(
fmt.Sprintf(
"http://%s:%s@realtime.oxylabs.io:60000",
apiCredentials.Username,
apiCredentials.Password,
),
)
if err != nil {
fmt.Printf("error parsing proxy url: %v", err)
}

// Prepare custom transport.
customTransport := &http.Transport{Proxy: http.ProxyURL(proxyUrl)}
customTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}

client := &http.Client{Transport: customTransport}

return &SerpClientProxy{
ApiCredentials: &ApiCredentials{
Username: username,
Password: password,
},
HttpClient: client,
}
}
114 changes: 0 additions & 114 deletions serp/proxy_endpint.go

This file was deleted.