filetail

Documentation

Overview

Copyright 2018 StreamSets Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Index

• Constants
• type FileInfo
• type FileTailConfigBean
• type FileTailOrigin
  • func (f *FileTailOrigin) Destroy() error
  • func (f *FileTailOrigin) Init(stageContext api.StageContext) []validation.Issue
  • func (f *FileTailOrigin) Produce(lastSourceOffset *string, maxBatchSize int, batchMaker api.BatchMaker) (*string, error)

Constants

const (
	Library                    = "streamsets-datacollector-basic-lib"
	StageName                  = "com_streamsets_pipeline_stage_origin_logtail_FileTailDSource"
	ConfGroupFiles             = "FILES"
	ConfFileInfos              = "conf.fileInfos"
	ConfMaxWaitTimeSecs        = "conf.maxWaitTimeSecs"
	ConfBatchSize              = "conf.batchSize"
	ConfDataFormat             = "conf.dataFormat"
	ErrorTail20                = "File path cannot be null or empty"
	ErrorTail02                = "File path doesn't exist: %s"
	ErrorTail08                = "The configuration for '%s' requires the '%s' token in the '%s' file name"
	FileRollModeReverseCounter = "REVERSE_COUNTER"
	FileRollModePattern        = "PATTERN"
	ELPattern                  = "${PATTERN}"
	FileAttribute              = "file"
	FileNameAttribute          = "filename"
	MTimeAttribute             = "mtime"
)

Types

type FileInfo

type FileInfo struct {
	FileFullPath    string `ConfigDef:"type=STRING,required=true"`
	FileRollMode    string `ConfigDef:"type=STRING,required=true"`
	PatternForToken string `ConfigDef:"type=STRING,required=true"`
	FirstFile       string `ConfigDef:"type=STRING,required=true"`
}

type FileTailConfigBean

type FileTailConfigBean struct {
	BatchSize        float64                           `ConfigDef:"type=NUMBER,required=true"`
	MaxWaitTimeSecs  float64                           `ConfigDef:"type=NUMBER,required=true"`
	FileInfos        []FileInfo                        `ConfigDef:"type=MODEL" ListBeanModel:"name=fileInfos"`
	DataFormat       string                            `ConfigDef:"type=STRING,required=true"`
	DataFormatConfig dataparser.DataParserFormatConfig `ConfigDefBean:"dataFormatConfig"`
}

type FileTailOrigin

type FileTailOrigin struct {
	*common.BaseStage
	Conf FileTailConfigBean `ConfigDefBean:"name=conf"`
	// contains filtered or unexported fields
}

func (*FileTailOrigin) Destroy

func (f *FileTailOrigin) Destroy() error

func (*FileTailOrigin) Init

func (f *FileTailOrigin) Init(stageContext api.StageContext) []validation.Issue

func (*FileTailOrigin) Produce

func (f *FileTailOrigin) Produce(
	lastSourceOffset *string,
	maxBatchSize int,
	batchMaker api.BatchMaker,
) (*string, error)