graph

package
v0.0.0-...-df423d5 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Apr 26, 2024 License: Apache-2.0 Imports: 50 Imported by: 0

Documentation

Index

Constants

View Source 
const (

	// limit consumption rate for single-item GETs requests,
	// or delta-based multi-item GETs, or item content download requests.
	SingleGetOrDeltaLC = 1
	// delta queries without a delta token cost 2 units
	DeltaNoTokenLC = 2
	// limit consumption rate for anything permissions related
	PermissionsLC = 5
)
View Source 
const (
	AttachmentChunkSize = 4 * 1024 * 1024

	// CopyBufferSize is used for chunked upload
	// Microsoft recommends 5-10MB buffers
	// https://docs.microsoft.com/en-us/graph/api/driveitem-createuploadsession?view=graph-rest-1.0#best-practices
	CopyBufferSize = 5 * 1024 * 1024
)
View Source 
const (
	ApplicationThrottled errorCode = "ApplicationThrottled"
	// this authN error is a catch-all used by graph in a variety of cases:
	// users without licenses, bad jwts, missing account permissions, etc.
	AuthenticationError errorCode = "AuthenticationError"
	// on the other hand, authZ errors apply specifically to authenticated,
	// but unauthorized, user requests
	AuthorizationRequestDenied errorCode = "Authorization_RequestDenied"

	ErrorAccessDenied errorCode = "ErrorAccessDenied"
	ErrorItemNotFound errorCode = "ErrorItemNotFound"
	// This error occurs when an email is enumerated but retrieving it fails
	// - we believe - due to it pre-dating mailbox creation. Possible explanations
	// are mailbox creation racing with email receipt or a similar issue triggered
	// due to on-prem->M365 mailbox migration.
	ErrorInvalidRecipients errorCode = "ErrorInvalidRecipients"
	// We have seen this graph error with a 500 response while backing up email
	// messages. It implies that the email message is corrupt and cannot be read.
	// Associated error message goes like "Data is corrupt.,Invalid global object ID:"
	ErrorCorruptData errorCode = "ErrorCorruptData"

	ItemNotFound                errorCode = "itemNotFound"
	MailboxNotEnabledForRESTAPI errorCode = "MailboxNotEnabledForRESTAPI"

	NotAllowed errorCode = "notAllowed"

	QuotaExceeded           errorCode = "ErrorQuotaExceeded"
	RequestResourceNotFound errorCode = "Request_ResourceNotFound"
	// Returned when we try to get the inbox of a user that doesn't exist.
	ResourceNotFound errorCode = "ResourceNotFound"
)
View Source 
const (
	IOErrDuringRead                 errorMessage = "IO error during request payload read"
	MysiteURLNotFound               errorMessage = "unable to retrieve user's mysite url"
	MysiteNotFound                  errorMessage = "user's mysite not found"
	NoSPLicense                     errorMessage = "Tenant does not have a SPO license"
	ParameterDeltaTokenNotSupported errorMessage = "Parameter 'DeltaToken' not supported for this request"
)
View Source 
const (
	LabelsMalware             = "malware_detected"
	LabelsMysiteNotFound      = "mysite_not_found"
	LabelsNoSharePointLicense = "no_sharepoint_license"

	// LabelsSkippable is used to determine if an error is skippable
	LabelsSkippable = "skippable_errors"
)
View Source 
const (
	// AddtlDataRemoved is the key value in the AdditionalData map
	// for when an item was deleted.
	//nolint:lll
	// https://learn.microsoft.com/en-us/graph/delta-query-overview?tabs=http#resource-representation-in-the-delta-query-response
	AddtlDataRemoved = "@removed"
)
View Source 
const JWTQueryParam = "tempauth"

JWTQueryParam is a query param embed in graph download URLs which holds JWT token.

View Source 
const (
	ResourceLocked errorCode = "resourceLocked"
)

inner error codes

Variables

View Source 
var (
	ErrServiceUnavailableEmptyResp = clues.New("service unavailable and no returned content")
	ErrNotFoundEmptyResp           = clues.New("not found and no returned content")
)

ErrServiceUnavailableEmptyResp indicates the remote service returned a 503 with an empty response body. This can sometimes happen if a request times out during processing.

TODO(ashmrtn): Either make a separate error struct for empty responses and implement Is() on it or start using tags on errors for the different status codes.

View Source 
var SafeURLPathParams = pii.MapWithPlurals(

	"alltime",
	"analytics",
	"archive",
	"beta",
	"calendargroup",
	"calendar",
	"calendarview",
	"channel",
	"childfolder",
	"children",
	"clone",
	"clutter",
	"column",
	"conflict",
	"contactfolder",
	"contact",
	"contenttype",
	"conversationhistory",
	"deleteditem",
	"delta",
	"draft",
	"drive",
	"event",
	"group",
	"inbox",
	"instance",
	"invitation",
	"item",
	"joinedteam",
	"junkemail",
	"label",
	"list",
	"localfailure",
	"mailfolder",
	"member",
	"message",
	"msgfolderroot",
	"notification",
	"outbox",
	"page",
	"primarychannel",
	"recoverableitemsdeletion",
	"root",
	"scheduled",
	"searchfolder",
	"security",
	"sentitem",
	"serverfailure",
	"site",
	"subscription",
	"syncissue",
	"team",
	"unarchive",
	"user",
	"v1.0")

well-known path names used by graph api calls used to un-hide path elements in a pii.SafeURL https://learn.microsoft.com/en-us/graph/api/resources/mailfolder?view=graph-rest-1.0

View Source 
var SafeURLQueryParams = map[string]struct{}{
	"deltatoken":    {},
	"startdatetime": {},
	"enddatetime":   {},
	"$count":        {},
	"$expand":       {},
	"$filter":       {},
	"$select":       {},
	"$top":          {},
}
well-known safe query parameters used by graph api calls

used to un-hide query params in a pii.SafeURL

Functions

func BaseCollections 

func BaseCollections(
	ctx context.Context,
	colls []data.BackupCollection,
	tenant, rOwner string,
	service path.ServiceType,
	categories map[path.CategoryType]struct{},
	su support.StatusUpdater,
	counter *count.Bus,
	errs *fault.Bus,
) ([]data.BackupCollection, error)

func BindRateLimiterConfig 

func BindRateLimiterConfig(ctx context.Context, lc LimiterCfg) context.Context

func CheckIDAndName 

func CheckIDAndName(c Container) error

CheckIDAndName is a validator that ensures the ID and name are populated and not zero valued.

func CheckIDNameAndParentFolderID 

func CheckIDNameAndParentFolderID(c Container) error

CheckIDNameAndParentFolderID is a validator that ensures the ID and name are populated and not zero valued.

func ConsumeNTokens 

func ConsumeNTokens(ctx context.Context, n int) context.Context

ConsumeNTokens ensures any calls using this context will consume n rate-limiter tokens. Default is 1, and this value does not need to be established in the context to consume the default tokens. This should only get used on a per-call basis, to avoid cross-pollination.

func CreateAdapter 

func CreateAdapter(
	tenant, client, secret string,
	counter *count.Bus,
	opts ...Option,
) (abstractions.RequestAdapter, error)

CreateAdapter uses provided credentials to log into M365 using Kiota Azure Library with Azure identity package. An adapter object is a necessary to component to create a graph api client connection.

func CreateGockAdapter 

func CreateGockAdapter(
	tenant, client, secret string,
	counter *count.Bus,
	opts ...Option,
) (abstractions.RequestAdapter, error)

CreateGockAdapter is similar to graph.CreateAdapter, but with option to enable interceptions via gock to make it mockable.

func GetAuth 

func GetAuth(tenant, client, secret string) (*kauth.AzureIdentityAuthenticationProvider, error)

func InitializeConcurrencyLimiter 

func InitializeConcurrencyLimiter(ctx context.Context, enable bool, capacity int)

func IsErrAccessDenied 

func IsErrAccessDenied(err error) bool

func IsErrAuthenticationError 

func IsErrAuthenticationError(err error) bool

func IsErrCannotOpenFileAttachment 

func IsErrCannotOpenFileAttachment(err error) bool

func IsErrConnectionReset 

func IsErrConnectionReset(err error) bool

func IsErrCorruptData 

func IsErrCorruptData(err error) bool

func IsErrDeltaNotSupported 

func IsErrDeltaNotSupported(err error) bool

func IsErrExchangeMailFolderNotFound 

func IsErrExchangeMailFolderNotFound(err error) bool

func IsErrFolderExists 

func IsErrFolderExists(err error) bool

func IsErrInvalidDelta 

func IsErrInvalidDelta(err error) bool

func IsErrInvalidRecipients 

func IsErrInvalidRecipients(err error) bool

func IsErrInvalidRequest 

func IsErrInvalidRequest(err error) bool

func IsErrQuotaExceeded 

func IsErrQuotaExceeded(err error) bool

func IsErrSharingDisabled 

func IsErrSharingDisabled(err error) bool

func IsErrSiteNotFound 

func IsErrSiteNotFound(err error) bool

func IsErrTimeout 

func IsErrTimeout(err error) bool

func IsErrUnauthorizedOrBadToken 

func IsErrUnauthorizedOrBadToken(err error) bool

func IsErrUsersCannotBeResolved 

func IsErrUsersCannotBeResolved(err error) bool

func IsMalware 

func IsMalware(err error) bool

IsMalware is true if the graphAPI returns a "malware detected" error code.

func IsMalwareResp 

func IsMalwareResp(ctx context.Context, resp *http.Response) bool

func IsURLExpired 

func IsURLExpired(
	ctx context.Context,
	urlStr string,
) (
	expiredErr error,
	err error,
)

IsURLExpired inspects the jwt token embed in the item download url and returns true if it is expired.

func ItemInfo 

func ItemInfo(item *custom.DriveItem) map[string]any

ItemInfo gathers potentially useful information about a drive item, and aggregates that data into a map.

func KiotaHTTPClient 

func KiotaHTTPClient(
	counter *count.Bus,
	opts ...Option,
) (*http.Client, *clientConfig)

KiotaHTTPClient creates a httpClient with middlewares and timeout configured for use in the graph adapter.

Re-use of http clients is critical, or else we leak OS resources and consume relatively unbound socket connections. It is important to centralize this client to be passed downstream where api calls can utilize it on a per-download basis.

func LabelStatus 

func LabelStatus(statusCode int) string

LabelStatus transforms the provided statusCode into a standard label that can be attached to a clues error and later reviewed when checking error statuses.

func LoggableURL 

func LoggableURL(url string) pii.SafeURL

func MakeMetadataCollection 

func MakeMetadataCollection(
	pathPrefix path.Path,
	metadata []MetadataCollectionEntry,
	statusUpdater support.StatusUpdater,
	counter *count.Bus,
) (data.BackupCollection, error)

MakeMetadataCollection creates a metadata collection that has a file containing all the provided metadata as a single json object. Returns nil if the map does not have any entries.

func NewAzureAuth 

func NewAzureAuth(creds account.M365Config) (*azureAuth, error)

func NewHTTPWrapper 

func NewHTTPWrapper(
	counter *count.Bus,
	opts ...Option,
) *httpWrapper

NewHTTPWrapper produces a http.Client wrapper that ensures calls use all the middleware we expect from the graph api client.

Re-use of http clients is critical, or else we leak OS resources and consume relatively unbound socket connections. It is important to centralize this client to be passed downstream where api calls can utilize it on a per-download basis.

func NewLargeItemWriter 

func NewLargeItemWriter(
	parentID, url string,
	size int64,
	counter *count.Bus,
) *largeItemWriter

func NewNoTimeoutHTTPWrapper 

func NewNoTimeoutHTTPWrapper(
	counter *count.Bus,
	opts ...Option,
) *httpWrapper

NewNoTimeoutHTTPWrapper constructs a http wrapper with no context timeout.

Re-use of http clients is critical, or else we leak OS resources and consume relatively unbound socket connections. It is important to centralize this client to be passed downstream where api calls can utilize it on a per-download basis.

func NewPrefixCollection 

func NewPrefixCollection(
	prev, full path.Path,
	su support.StatusUpdater,
	counter *count.Bus,
) (*prefixCollection, error)

Creates a new collection that only handles prefix pathing.

func Parallelism 

func Parallelism(srv path.ServiceType) parallelism

Parallelism returns the Parallelism for the requested service.

func QueueRequest 

func QueueRequest(ctx context.Context)

QueueRequest will allow the request to occur immediately if we're under the calls-per-minute rate. Otherwise, the call will wait in a queue until the next token set is available.

func ResetLimiter 

func ResetLimiter(ctx context.Context)

ResetLimiter resets the limiter to its initial state and refills tokens to initial capacity. This is only relevant for the sliding window limiter, and a no-op for token bucket limiter. The token bucket limiter doesn't need to be reset since it refills tokens at a fixed per-second rate.

func Stack 

func Stack(ctx context.Context, e error) *clues.Err

Stack is a helper function that extracts ODataError metadata from the error. If the error is not an ODataError type, returns the error. You probably don't need this, because all calls to the graph client automatically Stack these details. You probably want clues.Stack. This is primarily exported for test helpers.

func Wrap 

func Wrap(ctx context.Context, e error, msg string) *clues.Err

Wrap is a helper function that extracts ODataError metadata from the error. If the error is not an ODataError type, returns the error. You probably don't need this, because all calls to the graph client automatically Stack these details. You probably want clues.Wrap. This is primarily exported for test helpers.

Types

type CacheFolder 

type CacheFolder struct {
	Container
	// contains filtered or unexported fields
}

func NewCacheFolder 

func NewCacheFolder(c Container, pb, lpb *path.Builder) CacheFolder

func (CacheFolder) Location 

func (cf CacheFolder) Location() *path.Builder

func (CacheFolder) Path 

func (cf CacheFolder) Path() *path.Builder

func (*CacheFolder) SetLocation 

func (cf *CacheFolder) SetLocation(newLocation *path.Builder)

func (*CacheFolder) SetPath 

func (cf *CacheFolder) SetPath(newPath *path.Builder)

type CachedContainer 

type CachedContainer interface {
	Container

	// Location contains either the display names for the dirs (if this is a calendar)
	// or nil
	Location() *path.Builder
	SetLocation(*path.Builder)

	// Path contains either the ids for the dirs (if this is a calendar)
	// or the display names for the dirs
	Path() *path.Builder
	SetPath(*path.Builder)
}

CachedContainer is used for local unit tests but also makes it so that this code can be broken into generic- and service-specific chunks later on to reuse logic in IDToPath.

type CalendarDisplayable 

type CalendarDisplayable struct {
	models.Calendarable
	// contains filtered or unexported fields
}

CalendarDisplayable is a transformative struct that aligns models.Calendarable interface with the container interface. Calendars do not have the 2 of the

func CreateCalendarDisplayable 

func CreateCalendarDisplayable(entry any, parentID string) *CalendarDisplayable

CreateCalendarDisplayable helper function to create the calendarDisplayable during msgraph-sdk-go iterative process @param entry is the input supplied by pageIterator.Iterate() @param parentID of Calendar sets. Only populate when used with EventCalendarCache

func (CalendarDisplayable) GetDisplayName 

func (c CalendarDisplayable) GetDisplayName() *string

GetDisplayName returns the *string of the calendar name

func (CalendarDisplayable) GetParentFolderId 

func (c CalendarDisplayable) GetParentFolderId() *string

GetParentFolderId returns the default calendar name address EventCalendars have a flat hierarchy and Calendars are rooted at the default

type Container 

type Container interface {
	GetIDer
	GetParentFolderIDer
	GetDisplayNamer
}

type ContainerResolver 

type ContainerResolver interface {
	// IDToPath takes an m365 container ID and converts it to a hierarchical path
	// to that container. The path has a similar format to paths on the local
	// file system.
	IDToPath(ctx context.Context, m365ID string) (*path.Builder, *path.Builder, error)

	// Populate performs initialization steps for the resolver
	// @param ctx is necessary param for Graph API tracing
	// @param baseFolderID represents the M365ID base that the resolver will
	// conclude its search. Default input is "".
	Populate(ctx context.Context, errs *fault.Bus, baseFolderID string, baseContainerPath ...string) error

	// PathInCache performs a look up of a path representation
	// and returns the m365ID of directory iff the pathString
	// matches the path of a container within the cache.
	// @returns bool represents if m365ID was found.
	PathInCache(pathString string) (string, bool)
	// LocationInCache performs a look up of a path representation
	// and returns the m365ID of directory iff the pathString
	// matches the logical path of a container within the cache.
	// @returns bool represents if m365ID was found.
	LocationInCache(pathString string) (string, bool)

	AddToCache(ctx context.Context, m365Container Container) error

	// ItemByID returns the container with the given ID if it's in the container
	// resolver. If the item isn't in the resolver then it returns nil. Assumes
	// resolved IDs are used not well-known names for containers.
	ItemByID(id string) CachedContainer

	// Items returns the containers in the cache.
	Items() []CachedContainer
}

ContainerResolver houses functions for getting information about containers from remote APIs (i.e. resolve folder paths with Graph API). Resolvers may cache information about containers.

type GetAdditionalDataer 

type GetAdditionalDataer interface {
	GetAdditionalData() map[string]any
}

type GetDeletedDateTimer 

type GetDeletedDateTimer interface {
	GetDeletedDateTime() *time.Time
}

type GetDisplayNamer 

type GetDisplayNamer interface {
	GetDisplayName() *string
}

type GetIDer 

type GetIDer interface {
	GetId() *string
}

type GetLastModifiedDateTimer 

type GetLastModifiedDateTimer interface {
	GetLastModifiedDateTime() *time.Time
}

type GetLastUpdatedDateTimer 

type GetLastUpdatedDateTimer interface {
	GetLastUpdatedDateTime() *time.Time
}

type GetParentFolderIDer 

type GetParentFolderIDer interface {
	GetParentFolderId() *string
}

type LimiterCfg 

type LimiterCfg struct {
	Service path.ServiceType
	// Experimental flag to enable sliding window rate limiter. It should only be
	// enabled for Exchange backups. It's set to false by default to prevent accidental
	// enablement for non backup operations and other services.
	EnableSlidingLimiter bool
}

type LoggingMiddleware 

type LoggingMiddleware struct{}

LoggingMiddleware can be used to log the http request sent by the graph client

func (*LoggingMiddleware) Intercept 

func (mw *LoggingMiddleware) Intercept(
	pipeline khttp.Pipeline,
	middlewareIndex int,
	req *http.Request,
) (*http.Response, error)

type MetadataCollection 

type MetadataCollection struct {
	// contains filtered or unexported fields
}

MetadataCollection in a simple collection that assumes all items to be returned are already resident in-memory and known when the collection is created. This collection has no logic for lazily fetching item data.

func NewMetadataCollection 

func NewMetadataCollection(
	p path.Path,
	items []metadataItem,
	statusUpdater support.StatusUpdater,
	counter *count.Bus,
) *MetadataCollection

func (MetadataCollection) DoNotMergeItems 

func (md MetadataCollection) DoNotMergeItems() bool

func (MetadataCollection) FullPath 

func (md MetadataCollection) FullPath() path.Path

func (MetadataCollection) Items 

func (md MetadataCollection) Items(
	ctx context.Context,
	_ *fault.Bus,
) <-chan data.Item

func (MetadataCollection) PreviousPath 

func (md MetadataCollection) PreviousPath() path.Path

TODO(ashmrtn): Fill in with previous path once the Controller compares old and new folder hierarchies.

func (MetadataCollection) State 

func (md MetadataCollection) State() data.CollectionState

TODO(ashmrtn): Fill in once the Controller compares old and new folder hierarchies.

type MetadataCollectionEntry 

type MetadataCollectionEntry struct {
	// contains filtered or unexported fields
}

MetadataCollectionEntry describes a file that should get added to a metadata collection. The Data value will be encoded into json as part of a transformation into a MetadataItem.

func NewMetadataEntry 

func NewMetadataEntry(fileName string, mData any) MetadataCollectionEntry

type MetricsMiddleware 

type MetricsMiddleware struct {
	// contains filtered or unexported fields
}

MetricsMiddleware aggregates per-request metrics on the events bus

func (*MetricsMiddleware) Intercept 

func (mw *MetricsMiddleware) Intercept(
	pipeline khttp.Pipeline,
	middlewareIndex int,
	req *http.Request,
) (*http.Response, error)

type Option 

type Option func(*clientConfig)

func AuthorizeRequester 

func AuthorizeRequester(a authorizer) Option

func MaxConnectionRetries 

func MaxConnectionRetries(max int) Option

func MaxRetries 

func MaxRetries(max int) Option

func MinimumBackoff 

func MinimumBackoff(min time.Duration) Option

func NoTimeout 

func NoTimeout() Option

NoTimeout sets the httpClient.Timeout to 48 hours (eg: unlimited). The resulting client isn't suitable for most queries, due to the capacity for a call to persist forever. This configuration should only be used when downloading very large files.

func Timeout 

func Timeout(timeout time.Duration) Option

type QueryParams 

type QueryParams struct {
	Category          path.CategoryType
	ProtectedResource idname.Provider
	TenantID          string
}

type RateLimiterMiddleware 

type RateLimiterMiddleware struct{}

RateLimiterMiddleware is used to ensure we don't overstep per-min request limits.

func (*RateLimiterMiddleware) Intercept 

func (mw *RateLimiterMiddleware) Intercept(
	pipeline khttp.Pipeline,
	middlewareIndex int,
	req *http.Request,
) (*http.Response, error)

type Requester 

type Requester interface {
	Request(
		ctx context.Context,
		method, url string,
		body io.Reader,
		headers map[string]string,
		requireAuth bool,
	) (*http.Response, error)
}

type RetryMiddleware 

type RetryMiddleware struct {
	// The maximum number of times a request can be retried
	MaxRetries int
	// The delay in seconds between retries
	Delay time.Duration
}

RetryMiddleware handles transient HTTP responses and retries the request given the retry options

func (RetryMiddleware) Intercept 

func (mw RetryMiddleware) Intercept(
	pipeline khttp.Pipeline,
	middlewareIndex int,
	req *http.Request,
) (*http.Response, error)

Intercept implements the interface and evaluates whether to retry a failed request.

type Service 

type Service struct {
	// contains filtered or unexported fields
}

func NewGockService 

func NewGockService(
	creds account.M365Config,
	counter *count.Bus,
	opts ...Option,
) (*Service, error)

func NewService 

func NewService(adapter abstractions.RequestAdapter) *Service

func (Service) Adapter 

func (s Service) Adapter() abstractions.RequestAdapter

func (Service) Client 

func (s Service) Client() *msgraphsdkgo.GraphServiceClient

func (Service) Serialize 

func (s Service) Serialize(object serialization.Parsable) ([]byte, error)

Seraialize writes an M365 parsable object into a byte array using the built-in application/json writer within the adapter.

type Servicer 

type Servicer interface {
	// Client() returns msgraph Service client that can be used to process and execute
	// the majority of the queries to the M365 Backstore
	Client() *msgraphsdkgo.GraphServiceClient
	// Adapter() returns GraphRequest adapter used to process large requests, create batches
	// and page iterators
	Adapter() abstractions.RequestAdapter
}

Source Files

View all Source files

Directories

Path Synopsis
models
sites

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.