http

Documentation

Index

• Variables
• func GetCookieString(jar http.CookieJar, urls string) (string, error)
• func NewJar() (http.CookieJar, error)
• func SetCookies(jar http.CookieJar, urls string, cs string) error

Variables

var CloseResponse = rnt.NewSimpleFunction("CloseResponse", []string{"res"},
	func(args []rnt.Object) rnt.Object {
		res := rnt.UnwrapNative(args[0]).(*http.Response)
		res.Body.Close()
		return rnt.None
	})

CloseResponse closes the response.

var Execute = rnt.NewSimpleFunction("Execute",
	[]string{"client", "method", "url", "data", "headers"},
	func(args []rnt.Object) rnt.Object {

		client := rnt.UnwrapNative(args[0]).(*http.Client)

		method := args[1].(rnt.Str).Value()
		url := args[2].(rnt.Str).Value()
		req, err := http.NewRequest(method, url, nil)
		if err != nil {
			return rnt.NewTuple(rnt.None, rnt.NewStr(err.Error()))
		}

		if args[4] != rnt.None {
			it := args[4].(rnt.Dict).Iterator()
			for e := it(); e != nil; e = it() {
				key := e.Key.(rnt.Str).Value()
				value := e.Value.(rnt.Str).Value()
				req.Header.Set(key, value)
			}

			req.Header.Del("Accept-Encoding")
		}

		if args[3] != rnt.None {
			data := args[3].(rnt.Bytes).Value()
			if len(data) > 0 {
				req.Body = ioutil.NopCloser(bytes.NewBuffer(data))
				req.ContentLength = int64(len(data))

				if req.Header.Get("Content-Type") == "" {
					req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
				}
			}
		}

		res, err := client.Do(req)
		if err != nil {
			return rnt.NewTuple(rnt.None, rnt.NewStr(err.Error()))
		}

		nativeRes := rnt.NewNative(res)
		resURL := rnt.NewStr(res.Request.URL.String())

		statusCode := rnt.NewInt(res.StatusCode)
		_status := res.Status
		if pref := fmt.Sprintf("%d ", res.StatusCode); strings.HasPrefix(_status, pref) {
			_status = _status[len(pref):]
		}
		status := rnt.NewStr(_status)

		_resHeaders := make(map[string]string)
		for key := range res.Header {
			value := res.Header.Get(key)
			_resHeaders[key] = value
		}
		resHeaders := rnt.DictLiteral(_resHeaders)

		return rnt.NewTuple(
			rnt.NewTuple(nativeRes, resURL, statusCode, status, resHeaders),
			rnt.None)
	})

Execute creates a request from the parameters, executes it, and returns a tuple of response info on success or an error message on failure.

var GetCookieHeader = rnt.NewSimpleFunction("GetCookieHeader",
	[]string{"jar", "url"},
	func(args []rnt.Object) rnt.Object {
		jar := rnt.UnwrapNative(args[0]).(http.CookieJar)
		urls := args[1].(rnt.Str).Value()
		cs, err := GetCookieString(jar, urls)
		if err != nil {
			panic(rnt.RaiseType(lib.ExtractorErrorType, err.Error()))
		}
		return rnt.NewStr(cs)
	})

GetCookieHeader returns the Cookie request header value for url.

var ParseCookieString = rnt.NewSimpleFunction("ParseCookieString",
	[]string{"cs"},
	func(args []rnt.Object) rnt.Object {
		cs := args[0].(rnt.Str).Value()
		cookies := parseCookieString(cs)
		res := make(map[string]string)
		for _, cookie := range cookies {
			res[cookie.Name] = cookie.Value
		}
		return rnt.DictLiteral(res)
	})

ParseCookieString parses cookies and returns a name-value map of them.

var ParseQS = rnt.NewFunction("parse_qs",
	[]rnt.Param{
		{Name: "qs"},
		{Name: "keep_blank_values", Def: rnt.False},
	}, 0, false, false,
	func(args []rnt.Object) rnt.Object {
		qs, st := lib.StringlikeVal(args[0])
		keepBlanks := rnt.IsTrue(args[1])
		values, err := url.ParseQuery(qs)
		if err != nil {
			panic(rnt.RaiseType(rnt.ValueErrorType, err.Error()))
		}
		resDict := rnt.NewDict()
		for name, vals := range values {
			resVals := []rnt.Object{}
			for _, val := range vals {
				if val != "" || keepBlanks {
					resVals = append(resVals, lib.NewStringlike(val, st))
				}
			}
			if len(resVals) > 0 {
				resDict.SetItem(rnt.NewStr(name), rnt.NewList(resVals...))
			}
		}
		return resDict
	})

ParseQS implements python/urllib.parse.parse_qs

var RandomIPv4 = rnt.NewSimpleFunction("RandomIP",
	[]string{"cidr"},
	func(args []rnt.Object) rnt.Object {
		cidr := args[0].(rnt.Str).Value()
		_, ipnet, err := net.ParseCIDR(cidr)
		if err != nil {
			panic(rnt.RaiseType(rnt.ValueErrorType, err.Error()))
		}
		ip := make(net.IP, net.IPv4len)
		binary.LittleEndian.PutUint32(ip, rand.Uint32())
		for i := range ip {
			ip[i] = ipnet.IP[i] | (ip[i] & ^ipnet.Mask[i])
		}
		return rnt.NewStr(ip.String())
	})

RandomIPv4 generates a random IPv4 address in the range denoted by cidr

var ReadResponseBody = rnt.NewSimpleFunction("ReadResponseBody", []string{"res"},
	func(args []rnt.Object) rnt.Object {
		res := rnt.UnwrapNative(args[0]).(*http.Response)
		defer res.Body.Close()
		bytes, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return rnt.NewTuple(rnt.None, rnt.NewStr(err.Error()))
		}
		return rnt.NewTuple(rnt.NewBytes(bytes...), rnt.None)
	})

ReadResponseBody reads the entire body of the response and closes it.

var SetCookie = rnt.NewSimpleFunction("SetCookie",
	[]string{"jar", "domain", "name", "value"},
	func(args []rnt.Object) rnt.Object {
		jar := rnt.UnwrapNative(args[0]).(http.CookieJar)
		domain := args[1].(rnt.Str).Value()
		name := args[2].(rnt.Str).Value()
		value := args[3].(rnt.Str).Value()

		u := &url.URL{
			Scheme: "http",
			Host:   domain,
		}

		c := &http.Cookie{
			Name:   name,
			Value:  value,
			Domain: domain,
		}
		jar.SetCookies(u, []*http.Cookie{c})
		return rnt.None
	})

SetCookie adds a new cookie to the jar

var URLEncode = rnt.NewFunction("urlencode",
	[]rnt.Param{
		{Name: "query"},
		{Name: "doseq", Def: rnt.False},
	}, 0, false, false,
	func(args []rnt.Object) rnt.Object {
		toString := func(o rnt.Object) string {
			if o.IsInstance(rnt.BytesType) {
				return string(o.(rnt.Bytes).Value())
			}
			return rnt.ToStr(o).Value()
		}
		doSeq := rnt.IsTrue(args[1])
		if args[0].IsInstance(rnt.DictType) {
			args[0] = rnt.Cal(rnt.GetAttr(args[0], "items", nil))
		}
		pairs := rnt.Cal(rnt.ListType, args[0]).(rnt.List).Elems()
		resValues := make(url.Values)
		for _, pair := range pairs {
			name := toString(rnt.GetItem(pair, rnt.NewInt(0)))
			val := rnt.GetItem(pair, rnt.NewInt(1))
			listVal := func() rnt.Object {
				defer rnt.Catch(rnt.TypeErrorType, nil, nil)
				if val.IsInstance(rnt.StrType) || val.IsInstance(rnt.BytesType) {

					return nil
				}
				return rnt.Cal(rnt.ListType, val)
			}()
			if listVal != nil && doSeq {
				for _, val := range listVal.(rnt.List).Elems() {
					resValues.Add(name, toString(val))
				}
			} else {
				resValues.Add(name, toString(val))
			}
		}
		return rnt.NewStr(resValues.Encode())
	})

URLEncode implements python/urllib.parse.urlencode

var URLJoin = rnt.NewSimpleFunction("urljoin", []string{"base", "url"},
	func(args []rnt.Object) rnt.Object {
		base := args[0].(rnt.Str).Value()
		urlv := args[1].(rnt.Str).Value()
		baseURL, err := url.Parse(base)
		if err != nil {
			return args[1]
		}
		resultURL, err := baseURL.Parse(urlv)
		if err != nil {
			return args[1]
		}
		return rnt.NewStr(resultURL.String())
	})

URLJoin implements approximately to python/urllib.parse.urljoin

var URLParse = rnt.NewSimpleFunction("urlparse", []string{"urlstring"},
	func(args []rnt.Object) rnt.Object {
		urlv := args[0].(rnt.Str).Value()
		parsedURL, err := url.Parse(urlv)
		if err != nil {
			panic(rnt.RaiseType(rnt.ValueErrorType, fmt.Sprintf(
				"failed to parse url: '%s'", urlv)))
		}
		escPath := parsedURL.EscapedPath()
		parameters := ""
		if pos := strings.LastIndex(escPath, ";"); pos >= 0 {
			parameters = escPath[pos+1:]
			escPath = escPath[:pos]
		}
		return rnt.NewTuple(
			rnt.NewStr(parsedURL.Scheme),
			rnt.NewStr(parsedURL.Host),
			rnt.NewStr(escPath),
			rnt.NewStr(parameters),
			rnt.NewStr(parsedURL.RawQuery),
			rnt.NewStr(parsedURL.Fragment),
		)
	})

URLParse implements python/urllib.parse.urlparse

var URLUnparse = rnt.NewSimpleFunction("urlunparse", []string{"parts"},
	func(args []rnt.Object) rnt.Object {
		parts := rnt.Cal(rnt.ListType, args[0]).(rnt.List).Elems()
		if len(parts) != 6 {
			panic(rnt.RaiseType(rnt.ValueErrorType, "expected 6 values to unpack"))
		}

		u := new(url.URL)
		u.Scheme = parts[0].(rnt.Str).Value()
		u.Host = parts[1].(rnt.Str).Value()
		u.RawPath = parts[2].(rnt.Str).Value()
		if params := parts[3].(rnt.Str).Value(); params != "" {
			u.RawPath += ";" + params
		}
		path, err := url.PathUnescape(u.RawPath)
		if err != nil {
			panic(rnt.RaiseType(rnt.ValueErrorType, err.Error()))
		}
		u.Path = path
		u.RawQuery = parts[4].(rnt.Str).Value()
		u.Fragment = parts[5].(rnt.Str).Value()
		return rnt.NewStr(u.String())
	})

URLUnparse implements python/urllib.parse.urlunparse

var Unquote = rnt.NewSimpleFunction("unquote", []string{"string"},
	func(args []rnt.Object) rnt.Object {
		s := args[0].(rnt.Str).Value()
		res, err := url.PathUnescape(s)
		if err != nil {
			panic(rnt.RaiseType(lib.ExtractorErrorType, err.Error()))
		}
		return rnt.NewStr(res)
	})

Unquote implements python/urllib.parse.unquote

var UnquotePlus = rnt.NewSimpleFunction("unquote_plus", []string{"string"},
	func(args []rnt.Object) rnt.Object {
		s := args[0].(rnt.Str).Value()
		res, err := url.QueryUnescape(s)
		if err != nil {
			panic(rnt.RaiseType(lib.ExtractorErrorType, err.Error()))
		}
		return rnt.NewStr(res)
	})

UnquotePlus implements python/urllib.parse.unquote_plus

Functions

func GetCookieString

func GetCookieString(jar http.CookieJar, urls string) (string, error)

GetCookieString returns a string of all cookies for urls.

func NewJar

func NewJar() (http.CookieJar, error)

NewJar creates a new cookie jar

func SetCookies

func SetCookies(jar http.CookieJar, urls string, cs string) error

SetCookies parses the cookie string and adds the resulting cookies (corresponding to the domain of urls) to the jar