archival

Documentation

Overview

Package archival contains data formats used for archival.

See https://github.com/ooni/spec.

Index

• Variables
• func NewFailedOperation(err error) *string
• func NewFailure(err error) *string
• type DNSAnswerEntry
• type DNSQueryEntry
  • func NewDNSQueriesList(begin time.Time, events []trace.Event) []DNSQueryEntry
• type ExtSpec
  • func (spec ExtSpec) AddTo(m *model.Measurement)
• type HTTPBody
• type HTTPHeader
  • func (hh HTTPHeader) MarshalJSON() ([]byte, error)
  • func (hh *HTTPHeader) UnmarshalJSON(d []byte) error
• type HTTPRequest
• type HTTPResponse
• type HTTPTor
• type MaybeBinaryValue
  • func (hb MaybeBinaryValue) MarshalJSON() ([]byte, error)
  • func (hb *MaybeBinaryValue) UnmarshalJSON(d []byte) error
• type NetworkEvent
  • func NewNetworkEventsList(begin time.Time, events []trace.Event) []NetworkEvent
• type RequestEntry
  • func NewRequestList(begin time.Time, events []trace.Event) []RequestEntry
• type TCPConnectEntry
  • func NewTCPConnectList(begin time.Time, events []trace.Event) []TCPConnectEntry
• type TCPConnectStatus
• type TLSHandshake
  • func NewTLSHandshakesList(begin time.Time, events []trace.Event) []TLSHandshake

Variables

var (
	// ExtDNS is the version of df-002-dnst.md
	ExtDNS = ExtSpec{Name: "dnst", V: 0}

	// ExtNetevents is the version of df-008-netevents.md
	ExtNetevents = ExtSpec{Name: "netevents", V: 0}

	// ExtHTTP is the version of df-001-httpt.md
	ExtHTTP = ExtSpec{Name: "httpt", V: 0}

	// ExtTCPConnect is the version of df-005-tcpconnect.md
	ExtTCPConnect = ExtSpec{Name: "tcpconnect", V: 0}

	// ExtTLSHandshake is the version of df-006-tlshandshake.md
	ExtTLSHandshake = ExtSpec{Name: "tlshandshake", V: 0}

	// ExtTunnel is the version of df-009-tunnel.md
	ExtTunnel = ExtSpec{Name: "tunnel", V: 0}
)

Functions

func NewFailedOperation

func NewFailedOperation(err error) *string

NewFailedOperation creates a failed operation string from the given error.

func NewFailure

func NewFailure(err error) *string

NewFailure creates a failure nullable string from the given error

Types

type DNSAnswerEntry

type DNSAnswerEntry struct {
	ASN        int64   `json:"asn,omitempty"`
	ASOrgName  string  `json:"as_org_name,omitempty"`
	AnswerType string  `json:"answer_type"`
	Hostname   string  `json:"hostname,omitempty"`
	IPv4       string  `json:"ipv4,omitempty"`
	IPv6       string  `json:"ipv6,omitempty"`
	TTL        *uint32 `json:"ttl"`
}

DNSAnswerEntry is the answer to a DNS query

type DNSQueryEntry

type DNSQueryEntry struct {
	Answers          []DNSAnswerEntry `json:"answers"`
	DialID           int64            `json:"dial_id,omitempty"`
	Engine           string           `json:"engine"`
	Failure          *string          `json:"failure"`
	Hostname         string           `json:"hostname"`
	QueryType        string           `json:"query_type"`
	ResolverHostname *string          `json:"resolver_hostname"`
	ResolverPort     *string          `json:"resolver_port"`
	ResolverAddress  string           `json:"resolver_address"`
	T                float64          `json:"t"`
	TransactionID    int64            `json:"transaction_id,omitempty"`
}

DNSQueryEntry is a DNS query with possibly an answer

func NewDNSQueriesList

func NewDNSQueriesList(begin time.Time, events []trace.Event) []DNSQueryEntry

NewDNSQueriesList returns a list of DNS queries.

type ExtSpec

type ExtSpec struct {
	Name string // extension name
	V    int64  // extension version
}

ExtSpec describes a data format extension

func (ExtSpec) AddTo

func (spec ExtSpec) AddTo(m *model.Measurement)

AddTo adds the current ExtSpec to the specified measurement

type HTTPBody

type HTTPBody = MaybeBinaryValue

HTTPBody is an HTTP body. As an implementation note, this type must be an alias for the MaybeBinaryValue type, otherwise the specific serialisation mechanism implemented by MaybeBinaryValue is not working.

type HTTPHeader

type HTTPHeader struct {
	Key   string
	Value MaybeBinaryValue
}

HTTPHeader is a single HTTP header.

func (HTTPHeader) MarshalJSON

func (hh HTTPHeader) MarshalJSON() ([]byte, error)

MarshalJSON marshals a single HTTP header to a tuple where the first element is a string and the second element is maybe-binary data.

func (*HTTPHeader) UnmarshalJSON

func (hh *HTTPHeader) UnmarshalJSON(d []byte) error

UnmarshalJSON is the opposite of MarshalJSON.

type HTTPRequest

type HTTPRequest struct {
	Body            HTTPBody                    `json:"body"`
	BodyIsTruncated bool                        `json:"body_is_truncated"`
	HeadersList     []HTTPHeader                `json:"headers_list"`
	Headers         map[string]MaybeBinaryValue `json:"headers"`
	Method          string                      `json:"method"`
	Tor             HTTPTor                     `json:"tor"`
	Transport       string                      `json:"x_transport"`
	URL             string                      `json:"url"`
}

HTTPRequest contains an HTTP request.

Headers are a map in Web Connectivity data format but we have added support for a list since January 2020.

type HTTPResponse

type HTTPResponse struct {
	Body            HTTPBody                    `json:"body"`
	BodyIsTruncated bool                        `json:"body_is_truncated"`
	Code            int64                       `json:"code"`
	HeadersList     []HTTPHeader                `json:"headers_list"`
	Headers         map[string]MaybeBinaryValue `json:"headers"`

	// The following fields are not serialised but are useful to simplify
	// analysing the measurements in telegram, whatsapp, etc.
	Locations []string `json:"-"`
}

HTTPResponse contains an HTTP response.

Headers are a map in Web Connectivity data format but we have added support for a list since January 2020.

type HTTPTor

type HTTPTor struct {
	ExitIP   *string `json:"exit_ip"`
	ExitName *string `json:"exit_name"`
	IsTor    bool    `json:"is_tor"`
}

HTTPTor contains Tor information

type MaybeBinaryValue

type MaybeBinaryValue struct {
	Value string
}

MaybeBinaryValue is a possibly binary string. We use this helper class to define a custom JSON encoder that allows us to choose the proper representation depending on whether the Value field is valid UTF-8 or not.

func (MaybeBinaryValue) MarshalJSON

func (hb MaybeBinaryValue) MarshalJSON() ([]byte, error)

MarshalJSON marshals a string-like to JSON following the OONI spec that says that UTF-8 content is represened as string and non-UTF-8 content is instead represented using `{"format":"base64","data":"..."}`.

func (*MaybeBinaryValue) UnmarshalJSON

func (hb *MaybeBinaryValue) UnmarshalJSON(d []byte) error

UnmarshalJSON is the opposite of MarshalJSON.

type NetworkEvent

type NetworkEvent struct {
	Address       string   `json:"address,omitempty"`
	ConnID        int64    `json:"conn_id,omitempty"`
	DialID        int64    `json:"dial_id,omitempty"`
	Failure       *string  `json:"failure"`
	NumBytes      int64    `json:"num_bytes,omitempty"`
	Operation     string   `json:"operation"`
	Proto         string   `json:"proto,omitempty"`
	T             float64  `json:"t"`
	Tags          []string `json:"tags,omitempty"`
	TransactionID int64    `json:"transaction_id,omitempty"`
}

NetworkEvent is a network event. It contains all the possible fields and most fields are optional. They are only added when it makes sense for them to be there _and_ we have data to show.

func NewNetworkEventsList

func NewNetworkEventsList(begin time.Time, events []trace.Event) []NetworkEvent

NewNetworkEventsList returns a list of DNS queries.

type RequestEntry

type RequestEntry struct {
	Failure       *string      `json:"failure"`
	Request       HTTPRequest  `json:"request"`
	Response      HTTPResponse `json:"response"`
	T             float64      `json:"t"`
	TransactionID int64        `json:"transaction_id,omitempty"`
}

RequestEntry is one of the entries that are part of the "requests" key of a OONI report.

func NewRequestList

func NewRequestList(begin time.Time, events []trace.Event) []RequestEntry

NewRequestList returns the list for "requests"

type TCPConnectEntry

type TCPConnectEntry struct {
	ConnID        int64            `json:"conn_id,omitempty"`
	DialID        int64            `json:"dial_id,omitempty"`
	IP            string           `json:"ip"`
	Port          int              `json:"port"`
	Status        TCPConnectStatus `json:"status"`
	T             float64          `json:"t"`
	TransactionID int64            `json:"transaction_id,omitempty"`
}

TCPConnectEntry contains one of the entries that are part of the "tcp_connect" key of a OONI report.

func NewTCPConnectList

func NewTCPConnectList(begin time.Time, events []trace.Event) []TCPConnectEntry

NewTCPConnectList creates a new TCPConnectList

type TCPConnectStatus

type TCPConnectStatus struct {
	Blocked *bool   `json:"blocked,omitempty"` // Web Connectivity only
	Failure *string `json:"failure"`
	Success bool    `json:"success"`
}

TCPConnectStatus contains the TCP connect status.

The Blocked field breaks the separation between measurement and analysis we have been enforcing for quite some time now. It is a legacy from the Web Connectivity experiment and it should be here because of that.

type TLSHandshake

type TLSHandshake struct {
	CipherSuite        string             `json:"cipher_suite"`
	ConnID             int64              `json:"conn_id,omitempty"`
	Failure            *string            `json:"failure"`
	NegotiatedProtocol string             `json:"negotiated_protocol"`
	NoTLSVerify        bool               `json:"no_tls_verify"`
	PeerCertificates   []MaybeBinaryValue `json:"peer_certificates"`
	ServerName         string             `json:"server_name"`
	T                  float64            `json:"t"`
	Tags               []string           `json:"tags"`
	TLSVersion         string             `json:"tls_version"`
	TransactionID      int64              `json:"transaction_id,omitempty"`
}

TLSHandshake contains TLS handshake data

func NewTLSHandshakesList

func NewTLSHandshakesList(begin time.Time, events []trace.Event) []TLSHandshake

NewTLSHandshakesList creates a new TLSHandshakesList