alluxio

package
v1.0.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Apr 18, 2024 License: Apache-2.0 Imports: 49 Imported by: 0

Documentation

Overview

Copyright 2023 The Fluid Author.

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at 

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Index

Constants

View Source 
const (
	// NON_NATIVE_MOUNT_DATA_NAME also used in master 'statefulset.yaml' and config 'alluxio-mount.conf.yaml'
	NON_NATIVE_MOUNT_DATA_NAME = "mount.info"

	MountConfigStorage   = "ALLUXIO_MOUNT_CONFIG_STORAGE"
	ConfigmapStorageName = "configmap"
)

Variables

This section is empty.

Functions

func Build 

func Build(id string, ctx cruntime.ReconcileRequestContext) (base.Engine, error)

Build function builds the Alluxio Engine

func GetReservedPorts added in v0.6.0 

func GetReservedPorts(client client.Client) (ports []int, err error)

GetReservedPorts defines restoration logic for AlluxioRuntime

func IsMountWithConfigMap added in v1.0.0 

func IsMountWithConfigMap() bool

func Precheck added in v0.8.0 

func Precheck(client client.Client, key types.NamespacedName) (found bool, err error)

Precheck checks if the given key can be found in the current runtime types

Types

type APIGateway added in v0.6.0 

type APIGateway struct {
	Enabled bool  `json:"enabled,omitempty"`
	Ports   Ports `json:"ports,omitempty"`
}

type Affinity added in v0.3.0 

type Affinity struct {
	NodeAffinity *NodeAffinity `json:"nodeAffinity"`
}

type Alluxio 

type Alluxio struct {
	FullnameOverride string `json:"fullnameOverride"`

	common.ImageInfo `json:",inline"`
	common.UserInfo  `json:",inline"`

	NodeSelector map[string]string `json:"nodeSelector,omitempty"`
	JvmOptions   []string          `json:"jvmOptions,omitempty"`

	// Note: the mount secret options may be in here if existing direct mount root path in alluxio
	// So, alluxio master pod should handle with replacing the secret file path.
	Properties map[string]string `json:"properties,omitempty"`

	Master Master `json:"master,omitempty"`

	JobMaster JobMaster `json:"jobMaster,omitempty"`

	Worker Worker `json:"worker,omitempty"`

	JobWorker JobWorker `json:"jobWorker,omitempty"`

	Fuse Fuse `json:"fuse,omitempty"`

	APIGateway APIGateway `json:"apiGateway,omitempty"`

	TieredStore TieredStore `json:"tieredstore,omitempty"`

	Metastore Metastore `json:"metastore,omitempty"`

	Journal Journal `json:"journal,omitempty"`

	ShortCircuit ShortCircuit `json:"shortCircuit,omitempty"`

	UFSPaths []UFSPath `json:"ufsPaths,omitempty"`

	UFSVolumes []UFSVolume `json:"ufsVolumes,omitempty"`

	InitUsers common.InitUsers `json:"initUsers,omitempty"`

	Monitoring string `json:"monitoring,omitempty"`

	HadoopConfig HadoopConfig `json:"hadoopConfig,omitempty"`

	Tolerations []corev1.Toleration `json:"tolerations,omitempty"`

	PlacementMode string `json:"placement,omitempty"`

	RuntimeIdentity common.RuntimeIdentity `json:"runtimeIdentity"`

	Owner *common.OwnerReference `json:"owner,omitempty"`
}

The value yaml file

type AlluxioEngine 

type AlluxioEngine struct {
	Log logr.Logger
	client.Client

	MetadataSyncDoneCh chan base.MetadataSyncResult

	UnitTest bool

	*ctrl.Helper
	Recorder record.EventRecorder
	// contains filtered or unexported fields
}

AlluxioEngine implements the Engine interface.

func (*AlluxioEngine) AssignNodesToCache 

func (e *AlluxioEngine) AssignNodesToCache(desiredNum int32) (currentScheduleNum int32, err error)

AssignNodesToCache finds nodes to place the cache engine

func (*AlluxioEngine) BindToDataset added in v0.5.0 

func (e *AlluxioEngine) BindToDataset() (err error)

func (*AlluxioEngine) CheckAndUpdateRuntimeStatus 

func (e *AlluxioEngine) CheckAndUpdateRuntimeStatus() (ready bool, err error)

CheckAndUpdateRuntimeStatus checks the related runtime status and updates it.

func (*AlluxioEngine) CheckMasterReady 

func (e *AlluxioEngine) CheckMasterReady() (ready bool, err error)

CheckMasterReady checks if the master is ready

func (*AlluxioEngine) CheckRuntimeHealthy 

func (e *AlluxioEngine) CheckRuntimeHealthy() (err error)

CheckRuntimeHealthy checks the healthy of the runtime

func (*AlluxioEngine) CheckRuntimeReady added in v0.6.0 

func (e *AlluxioEngine) CheckRuntimeReady() (ready bool)

func (*AlluxioEngine) CheckWorkersReady 

func (e *AlluxioEngine) CheckWorkersReady() (ready bool, err error)

are the workers ready

func (*AlluxioEngine) CreateDataLoadJob added in v0.6.0 

func (e *AlluxioEngine) CreateDataLoadJob(ctx cruntime.ReconcileRequestContext, targetDataload datav1alpha1.DataLoad) (err error)

CreateDataLoadJob creates the job to load data

func (*AlluxioEngine) CreateDataMigrateJob added in v0.9.0 

func (e *AlluxioEngine) CreateDataMigrateJob(ctx cruntime.ReconcileRequestContext, targetDataMigrate datav1alpha1.DataMigrate) error

func (*AlluxioEngine) CreateVolume 

func (e *AlluxioEngine) CreateVolume() (err error)

CreateVolume creates volume

func (*AlluxioEngine) DeleteVolume 

func (e *AlluxioEngine) DeleteVolume() (err error)

DeleteVolume creates volume

func (*AlluxioEngine) FindUnmountedUFS added in v0.7.0 

func (e *AlluxioEngine) FindUnmountedUFS() (unmountedPaths []string, err error)

FindUnmountedUFS return if UFSs not mounted

func (*AlluxioEngine) FreeStorageBytes 

func (e *AlluxioEngine) FreeStorageBytes() (value int64, err error)

FreeStorageBytes returns free storage size of Alluxio in bytes

func (*AlluxioEngine) GetAPIGatewayStatus added in v0.6.0 

func (e *AlluxioEngine) GetAPIGatewayStatus() (status *datav1alpha1.APIGatewayStatus, err error)

Query the API Gateway status

func (*AlluxioEngine) GetCacheHitStates added in v0.6.0 

func (e *AlluxioEngine) GetCacheHitStates() (cacheHitStates cacheHitStates)

getCacheHitStates gets cache hit related info by parsing Alluxio metrics

func (*AlluxioEngine) GetDataOperationValueFile added in v0.9.0 

func (e *AlluxioEngine) GetDataOperationValueFile(ctx cruntime.ReconcileRequestContext, operation dataoperation.OperationInterface) (valueFileName string, err error)

func (*AlluxioEngine) GetHCFSStatus added in v0.4.0 

func (e *AlluxioEngine) GetHCFSStatus() (status *datav1alpha1.HCFSStatus, err error)

Query the hcfs status

func (*AlluxioEngine) GetMetadataFileName added in v0.5.0 

func (e *AlluxioEngine) GetMetadataFileName() string

func (*AlluxioEngine) GetMetadataInfoFile added in v0.5.0 

func (e *AlluxioEngine) GetMetadataInfoFile() string

func (*AlluxioEngine) GetMetadataInfoFileName added in v0.5.0 

func (e *AlluxioEngine) GetMetadataInfoFileName() string

func (*AlluxioEngine) GetReportMetrics added in v0.6.0 

func (e *AlluxioEngine) GetReportMetrics() (summary string, err error)

reportMetrics reports alluxio metrics

func (*AlluxioEngine) GetReportSummary added in v0.6.0 

func (e *AlluxioEngine) GetReportSummary() (summary string, err error)

reportSummary reports alluxio summary

func (*AlluxioEngine) GetWorkerUsedCapacity added in v0.5.0 

func (e *AlluxioEngine) GetWorkerUsedCapacity() (map[string]int64, error)

GetWorkerUsedCapacity gets cache capacity usage for each worker as a map. It parses result from stdout when executing `alluxio fsadmin report capacity` command and extracts worker name(IP or hostname) along with used capacity for that worker

func (*AlluxioEngine) HasDeprecatedCommonLabelname added in v0.6.0 

func (e *AlluxioEngine) HasDeprecatedCommonLabelname() (deprecated bool, err error)

func (AlluxioEngine) ParseReportMetric added in v0.6.0 

func (e AlluxioEngine) ParseReportMetric(metrics string, cacheHitStates, lastCacheHitStates *cacheHitStates)

parse alluxio report metric to cacheHitStates

func (AlluxioEngine) ParseReportSummary added in v0.6.0 

func (e AlluxioEngine) ParseReportSummary(s string) cacheStates

parse alluxio report summary to cacheStates

func (*AlluxioEngine) PrepareUFS 

func (e *AlluxioEngine) PrepareUFS() (err error)

PrepareUFS does all the UFS preparations

func (*AlluxioEngine) RestoreMetadataInternal added in v0.5.0 

func (e *AlluxioEngine) RestoreMetadataInternal() (err error)

RestoreMetadataInternal restores metadata from backup there are three kinds of data to be restored 1. metadata of dataset 2. ufsTotal info of dataset 3. fileNum info of dataset if 1 fails, the alluxio master will fail directly, if 2 or 3 fails, fluid will get the info from alluxio again

func (*AlluxioEngine) SetupMaster 

func (e *AlluxioEngine) SetupMaster() (err error)

SetupMaster setups the master and updates the status It will print the information in the Debug window according to the Master status It return any cache error encountered

func (*AlluxioEngine) SetupWorkers 

func (e *AlluxioEngine) SetupWorkers() (err error)

SetupWorkers checks the desired and current replicas of workers and makes an update over the status by setting phases and conditions. The function calls for a status update and finally returns error if anything unexpected happens.

func (*AlluxioEngine) ShouldCheckUFS 

func (e *AlluxioEngine) ShouldCheckUFS() (should bool, err error)

ShouldCheckUFS checks if it requires checking UFS

func (*AlluxioEngine) ShouldSetupMaster 

func (e *AlluxioEngine) ShouldSetupMaster() (should bool, err error)

ShouldSetupMaster checks if we need setup the master

func (*AlluxioEngine) ShouldSetupWorkers 

func (e *AlluxioEngine) ShouldSetupWorkers() (should bool, err error)

ShouldSetupWorkers checks if we need setup the workers

func (*AlluxioEngine) ShouldUpdateUFS added in v0.6.0 

func (e *AlluxioEngine) ShouldUpdateUFS() (ufsToUpdate *utils.UFSToUpdate)

func (*AlluxioEngine) Shutdown 

func (e *AlluxioEngine) Shutdown() (err error)

shut down the Alluxio engine

func (*AlluxioEngine) SyncMetadata added in v0.4.0 

func (e *AlluxioEngine) SyncMetadata() (err error)

SyncMetadata syncs metadata if necessary For Alluxio Engine, metadata sync is an asynchronous operation, which means you should call this function periodically to make sure the function actually takes effect.

func (*AlluxioEngine) SyncReplicas 

func (e *AlluxioEngine) SyncReplicas(ctx cruntime.ReconcileRequestContext) (err error)

SyncReplicas syncs the replicas

func (*AlluxioEngine) SyncRuntime added in v0.8.0 

func (e *AlluxioEngine) SyncRuntime(ctx cruntime.ReconcileRequestContext) (changed bool, err error)

SyncRuntime syncs the runtime spec

func (*AlluxioEngine) SyncScheduleInfoToCacheNodes added in v0.7.0 

func (e *AlluxioEngine) SyncScheduleInfoToCacheNodes() (err error)

SyncScheduleInfoToCacheNodes syncs schedule info to nodes

func (*AlluxioEngine) TotalFileNums 

func (e *AlluxioEngine) TotalFileNums() (value int64, err error)

TotalFileNums returns the total num of files in Alluxio

func (*AlluxioEngine) TotalStorageBytes 

func (e *AlluxioEngine) TotalStorageBytes() (value int64, err error)

TotalStorageBytes returns total storage size of Alluxio in bytes

func (*AlluxioEngine) UpdateCacheOfDataset 

func (e *AlluxioEngine) UpdateCacheOfDataset() (err error)

UpdateCacheOfDataset updates the CacheStates and Runtimes of the dataset.

func (*AlluxioEngine) UpdateDatasetStatus 

func (e *AlluxioEngine) UpdateDatasetStatus(phase datav1alpha1.DatasetPhase) (err error)

UpdateDatasetStatus updates the status of the dataset

func (*AlluxioEngine) UpdateOnUFSChange added in v0.6.0 

func (e *AlluxioEngine) UpdateOnUFSChange(ufsToUpdate *utils.UFSToUpdate) (updateReady bool, err error)

func (*AlluxioEngine) UsedStorageBytes 

func (e *AlluxioEngine) UsedStorageBytes() (value int64, err error)

UsedStorageBytes returns used storage size of Alluxio in bytes

type Fuse 

type Fuse struct {
	Image              string               `json:"image,omitempty"`
	NodeSelector       map[string]string    `json:"nodeSelector,omitempty"`
	ImageTag           string               `json:"imageTag,omitempty"`
	ImagePullPolicy    string               `json:"imagePullPolicy,omitempty"`
	Properties         map[string]string    `json:"properties,omitempty"`
	Env                map[string]string    `json:"env,omitempty"`
	JvmOptions         []string             `json:"jvmOptions,omitempty"`
	MountPath          string               `json:"mountPath,omitempty"`
	ShortCircuitPolicy string               `json:"shortCircuitPolicy,omitempty"`
	Args               []string             `json:"args,omitempty"`
	HostNetwork        bool                 `json:"hostNetwork,omitempty"`
	HostPID            bool                 `json:"hostPID,omitempty"`
	Enabled            bool                 `json:"enabled,omitempty"`
	Resources          common.Resources     `json:"resources,omitempty"`
	Global             bool                 `json:"global,omitempty"`
	CriticalPod        bool                 `json:"criticalPod,omitempty"`
	VolumeMounts       []corev1.VolumeMount `json:"volumeMounts,omitempty"`
	Volumes            []corev1.Volume      `json:"volumes,omitempty"`
	Labels             map[string]string    `json:"labels,omitempty"`
	Annotations        map[string]string    `json:"annotations,omitempty"`
}

type HadoopConfig added in v0.5.0 

type HadoopConfig struct {
	ConfigMap       string `json:"configMap"`
	IncludeHdfsSite bool   `json:"includeHdfsSite"`
	IncludeCoreSite bool   `json:"includeCoreSite"`
}

type JobMaster added in v0.4.0 

type JobMaster struct {
	Ports     Ports            `json:"ports,omitempty"`
	Resources common.Resources `json:"resources,omitempty"`
}

type JobWorker added in v0.4.0 

type JobWorker struct {
	Ports     Ports            `json:"ports,omitempty"`
	Resources common.Resources `json:"resources,omitempty"`
}

type Journal 

type Journal struct {
	VolumeType string `json:"volumeType,omitempty"`
	Size       string `json:"size,omitempty"`
}

type Level 

type Level struct {
	Alias      string `json:"alias,omitempty"`
	Level      int    `json:"level"`
	MediumType string `json:"mediumtype,omitempty"`
	Type       string `json:"type,omitempty"`
	Path       string `json:"path,omitempty"`
	Quota      string `json:"quota,omitempty"`
	High       string `json:"high,omitempty"`
	Low        string `json:"low,omitempty"`
}

type Master 

type Master struct {
	JvmOptions   []string             `json:"jvmOptions,omitempty"`
	Env          map[string]string    `json:"env,omitempty"`
	Affinity     Affinity             `json:"affinity"`
	NodeSelector map[string]string    `json:"nodeSelector,omitempty"`
	Properties   map[string]string    `json:"properties,omitempty"`
	Replicas     int32                `json:"replicaCount,omitempty"`
	HostNetwork  bool                 `json:"hostNetwork,omitempty"`
	Resources    common.Resources     `json:"resources,omitempty"`
	Ports        Ports                `json:"ports,omitempty"`
	BackupPath   string               `json:"backupPath,omitempty"`
	Restore      Restore              `json:"restore,omitempty"`
	VolumeMounts []corev1.VolumeMount `json:"volumeMounts,omitempty"`
	Volumes      []corev1.Volume      `json:"volumes,omitempty"`
	Labels       map[string]string    `json:"labels,omitempty"`
	Annotations  map[string]string    `json:"annotations,omitempty"`
	// config storage to store mount config.
	MountConfigStorage string `json:"mountConfigStorage,omitempty"`
	// non native mount infos when using configmap as mount storage.
	NonNativeMounts []string `json:"nonNativeMounts,omitempty"`
}

type Metastore 

type Metastore struct {
	VolumeType string `json:"volumeType,omitempty"`
	Size       string `json:"size,omitempty"`
}

type NodeAffinity added in v0.3.0 

type NodeAffinity struct {

	// If the affinity requirements specified by this field are not met at
	// scheduling time, the pod will not be scheduled onto the node.
	// If the affinity requirements specified by this field cease to be met
	// at some point during pod execution (e.g. due to an update), the system
	// may or may not try to eventually evict the pod from its node.
	// +optional
	RequiredDuringSchedulingIgnoredDuringExecution *NodeSelector `json:"requiredDuringSchedulingIgnoredDuringExecution"`
}

type NodeSelector added in v0.3.0 

type NodeSelector struct {
	//Required. A list of node selector terms.
	NodeSelectorTerms []NodeSelectorTerm `json:"nodeSelectorTerms"`
}

NodeSelector represents the union of the results of one or more label queries over a set of nodes; that is, it represents the OR of the selectors represented by the node selector terms.

type NodeSelectorRequirement added in v0.3.0 

type NodeSelectorRequirement struct {
	// The label key that the selector applies to.
	Key string `json:"key,omitempty"`
	// Represents a key's relationship to a set of values.
	// Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.
	Operator string `json:"operator,omitempty"`
	// +optional
	Values []string `json:"values,omitempty"`
}

NodeSelectorRequirement is a selector that contains values, a key, and an operator that relates the key and values.

type NodeSelectorTerm added in v0.3.0 

type NodeSelectorTerm struct {
	// A list of node selector requirements by node's labels.
	MatchExpressions []NodeSelectorRequirement `json:"matchExpressions"`
}

NodeSelectorTerm represents expressions and fields required to select nodes. A null or empty node selector term matches no objects. The requirements of them are ANDed. The TopologySelectorTerm type implements a subset of the NodeSelectorTerm.

type Ports added in v0.4.0 

type Ports struct {
	Rpc      int `json:"rpc,omitempty"`
	Web      int `json:"web,omitempty"`
	Embedded int `json:"embedded,omitempty"`
	Data     int `json:"data,omitempty"`
	Rest     int `json:"rest,omitempty"`
}

type Restore added in v0.5.0 

type Restore struct {
	Enabled bool   `json:"enabled,omitempty"`
	Path    string `json:"path,omitempty"`
	PVCName string `json:"pvcName,omitempty"`
}

type ShortCircuit 

type ShortCircuit struct {
	Enable     bool   `json:"enable,omitempty"`
	Policy     string `json:"policy,omitempty"`
	VolumeType string `json:"volumeType,omitempty"`
}

type TieredStore added in v0.6.0 

type TieredStore struct {
	Levels []Level `json:"levels,omitempty"`
}

type UFSPath added in v0.3.0 

type UFSPath struct {
	HostPath  string `json:"hostPath"`
	UFSVolume `json:",inline"`
}

type UFSVolume added in v0.3.0 

type UFSVolume struct {
	Name          string `json:"name"`
	SubPath       string `json:"subPath,omitempty"`
	ContainerPath string `json:"containerPath"`
}

type Worker 

type Worker struct {
	JvmOptions   []string             `json:"jvmOptions,omitempty"`
	Env          map[string]string    `json:"env,omitempty"`
	NodeSelector map[string]string    `json:"nodeSelector,omitempty"`
	Properties   map[string]string    `json:"properties,omitempty"`
	HostNetwork  bool                 `json:"hostNetwork,omitempty"`
	Resources    common.Resources     `json:"resources,omitempty"`
	Ports        Ports                `json:"ports,omitempty"`
	VolumeMounts []corev1.VolumeMount `json:"volumeMounts,omitempty"`
	Volumes      []corev1.Volume      `json:"volumes,omitempty"`
	Labels       map[string]string    `json:"labels,omitempty"`
	Annotations  map[string]string    `json:"annotations,omitempty"`
}

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.