snapsdb

package module
v1.0.4 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Oct 18, 2022 License: MIT Imports: 17 Imported by: 0

README

snapsdb

golang snapsort objects store database

snapsdb 是一款对象数据列表快照数据库，它诞生的目的是为了解决瞬间查询某些数据在历史上某一时刻的数据快照， 它使用 protobuf 作为对象的序列化方式，这意味着您的对象必须由protoc指令创建，这样带来的好处是序列化性能的大幅度提升。

snapsdb是以时间线为单位的，每天会生成一个单独的文件。 在这个文件的开头存储着当天86400秒的所有时间线索引，这个索引分别是 first last 两条记录，first负责数据查询读取，last负责新的数据写入。 这两个对象所指向的是一个单向链表，这样我们可以在任意时间存储任意时间线的数据。

⚠️ 这个数据库不支持索引， 不支持数据聚合，目前它仅完成了数据写入和数据查询的功能。

use library💎

 go get -u github.com/vblegend/snapsdb

📦 look look test code 🤝

package test

import (
	"fmt"
	"path/filepath"

	"testing"
	"time"

	"github.com/vblegend/snapsdb"
	"github.com/vblegend/snapsdb/test/types"
	"github.com/vblegend/snapsdb/util"

	"google.golang.org/protobuf/proto"
)

func InitDB() snapsdb.SnapsDB {
	snapPath := filepath.Join(util.AssemblyDir(), "../snapdata/proc")
	db, err := snapsdb.InitDB(
		snapsdb.WithDataPath(snapPath),
		snapsdb.WithDataRetention(snapsdb.TimestampOf14Day),
		snapsdb.WithTimeKeyFormat("2006-01-02 15:04:05"),
	)
	if err != nil {
		panic(err)
	}
	return db
}

// 测试 snapshotDB 批量插入
func TestSnapshotDBWrite(_ *testing.T) {
	fmt.Println("开始测试")
	db := InitDB()
	v1 := &types.ProcessInfo{Pid: 1, Name: "docker-compose - 1", Cpu: 10.01, Mem: 91.23, Virt: 10000000000, Res: 110000000000000}
	v2 := &types.ProcessInfo{Pid: 2, Name: "docker-compose - 2", Cpu: 20.02, Mem: 92.34, Virt: 20000000000, Res: 220000000000000}
	v3 := &types.ProcessInfo{Pid: 3, Name: "docker-compose - 3", Cpu: 30.03, Mem: 93.45, Virt: 30000000000, Res: 330000000000000}
	v4 := &types.ProcessInfo{Pid: 4, Name: "docker-compose - 4", Cpu: 40.04, Mem: 94.56, Virt: 40000000000, Res: 440000000000000}
	v5 := &types.ProcessInfo{Pid: 5, Name: "docker-compose - 5", Cpu: 50.05, Mem: 95.67, Virt: 50000000000, Res: 550000000000000}

	data1, _ := proto.Marshal(v1)
	count := 172800
	array := []snapsdb.StoreData{v1, v2, v3, v4, v5}

	start := time.Now() // 获取当前时间
	for i := 0; i < count; i++ {
		timestamp := time.Date(2022, 9, 22, 0, 0, i, 0, time.Local)
		db.Write(timestamp, array...)
	}
	cost := time.Since(start)
	total := count * len(array)
	bySec := float64(total) / cost.Seconds()
	fmt.Printf("写入完毕...\n共计写入%s条数据\n每条数据长度%s字节\n单位数据量%s条\n共用时%s\n每秒写入量%s条\n",
		util.Green(fmt.Sprintf("%d", total)),
		util.Green(fmt.Sprintf("%d", len(data1))),
		util.Green(fmt.Sprintf("%d", len(array))),
		util.Green(fmt.Sprintf("%v", cost)),
		util.Green(fmt.Sprintf("%.0f", bySec)))
}

// 测试 snapshotDB 单条插入
func TestSnapshotDBWriteOnce(t *testing.T) {
	fmt.Println("开始测试")
	db := InitDB()
	v1 := &types.ProcessInfo{Pid: 1, Name: "docker-compose - 1", Cpu: 10.01, Mem: 91.23, Virt: 10000000000, Res: 110000000000000}
	v2 := &types.ProcessInfo{Pid: 2, Name: "docker-compose - 2", Cpu: 20.02, Mem: 92.34, Virt: 20000000000, Res: 220000000000000}
	v3 := &types.ProcessInfo{Pid: 3, Name: "docker-compose - 3", Cpu: 30.03, Mem: 93.45, Virt: 30000000000, Res: 330000000000000}
	v4 := &types.ProcessInfo{Pid: 4, Name: "docker-compose - 4", Cpu: 40.04, Mem: 94.56, Virt: 40000000000, Res: 440000000000000}
	v5 := &types.ProcessInfo{Pid: 5, Name: "docker-compose - 5", Cpu: 50.05, Mem: 95.67, Virt: 50000000000, Res: 550000000000000}
	timestamp := time.Date(2022, 01, 01, 01, 01, 01, 01, time.Local)
	fmt.Println(timestamp.Format("2006-01-02 15:04:05"))
	start := time.Now() // 获取当前时间
	db.Write(timestamp, v1, v2, v3, v4, v5)
	cost := time.Since(start)
	fmt.Printf("写入完毕，共计%d条数据，用时%v\n", 5, cost)
}

// 测试 snapshotDB 的时间线查询
func TestSnapshotDBQuery(t *testing.T) {
	db := InitDB()
	timestamp := time.Date(2022, 9, 22, 13, 27, 43, 0, time.Local)
	list := make([]types.ProcessInfo, 0)
	list = append(list, types.ProcessInfo{Pid: 5, Name: "docker-compose - 1111", Cpu: 50.05, Mem: 95.67, Virt: 50000000000, Res: 550000000000000})
	start := time.Now()
	err := db.QueryTimeline(timestamp, &list)
	cost := time.Since(start)
	if err != nil {
		fmt.Println(util.Red(err.Error()))
	}
	fmt.Println(list)
	fmt.Printf("查询完毕，用时%v\n", cost)
}

// 测试 snapshotDB 的时间段查询
func TestSnapshotDBQueryBetween(t *testing.T) {
	db := InitDB()
	beginTimestamp := time.Date(2022, 9, 22, 5, 0, 00, 0, time.Local)
	endTimestamp := time.Date(2022, 9, 22, 5, 2, 00, 0, time.Local)
	outmap := make(map[string][]types.ProcessInfo)
	start := time.Now()
	err := db.QueryBetween(beginTimestamp, endTimestamp, &outmap)
	cost := time.Since(start)
	if err != nil && err != snapsdb.ErrorDBFileNotHit {
		fmt.Println(util.Red(err.Error()))
	}
	fmt.Println(outmap)
	fmt.Printf("查询完毕，用时%v\n", cost)
}

Documentation

Index

Constants

View Source 
const (
	// Timestamp length in 1 day
	TimestampOf1Day = time.Hour * 24
	// Timestamp length in 7 day
	TimestampOf7Day = TimestampOf1Day * 7
	// Timestamp length in 14 day
	TimestampOf14Day = TimestampOf1Day * 14
	// Timestamp length in 30 day
	TimestampOf30Day = TimestampOf1Day * 30
	// Timestamp length in 1 year
	TimestampOf1Year = TimestampOf1Day * 365
	// Timestamp length in 100 year
	TimestampOf100Year = TimestampOf1Year * 100
)

time

View Source 
const (
	// 一天的时间线长度
	TimelineLengthOfDay = int64(86400)
	// 文件头的偏移量
	FileHeaderOffset = int64(16)
	// 单条时间线元数据的大小
	MateInfoSize = int64(8)
	// 一天的时间线元数据总大小
	MateTableSize = int64(691200)
	// 下一条数据记录的指针偏移位置（相对于数据记录的开始位置）
	NextDataOffset = int64(8)
	// 文件第一条数据的偏移位置
	FileDataOffset = uint32(MateTableSize + FileHeaderOffset)
	// timeline    8 byte
	// nextdata    4 byte
	// datalen     4 byte
	DataHeaderLen = 8 + 4 + 4
)

stroage file

Variables

View Source 
var ErrorDBFileNotHit = errors.New("one or more files were not hit(not found datastore file).")

Functions

func Traverse added in v1.0.4 

func Traverse(object any, name string)

func TraverseBinary added in v1.0.4 

func TraverseBinary(value []byte, name string)

func TraverseMap added in v1.0.4 

func TraverseMap(mv map[string]interface{}, name string)

func TraverseStruct added in v1.0.4 

func TraverseStruct(v reflect.Value, name string)

func WriteVlaue added in v1.0.4 

func WriteVlaue(name string, value any)

Types

type Binary added in v1.0.4 

type Binary []byte

type DataPoint added in v1.0.4 

type DataPoint struct {
	Tags   TagPair
	Values ValuePair
}

snapsdb data point

type HashMap added in v1.0.4 

type HashMap map[string]interface{}

type Option 

type Option func(*dbOptions)

func WithDataPath 

func WithDataPath(dataPath string) Option

data storage path, do not duplicate with other libraries. default("./data")

func WithDataRetention 

func WithDataRetention(value time.Duration) Option

Data storage strategy, how long to save. default(TimestampOf7Day)

func WithTimeKeyFormat added in v1.0.2 

func WithTimeKeyFormat(value string) Option

When the map key is a string, the time format of the key. default("2006-01-02 15:04:05")

type SnapsDB 

type SnapsDB interface {
	// write one or more pieces of data to the timeline.
	Write(timeline time.Time, data ...StoreData) error
	WriteUnix(timeline int64, data ...StoreData) error
	// Query a certain timeline data, and return to the slice
	// the slice type should be inherited from protoreflect.ProtoMessage
	/*
		@example
		timestamp := time.Date(2020, 9, 22, 13, 27, 43, 0, time.Local)
		list := make([]types.ProcessInfo, 0)
		db.QueryTimeline(timestamp, &list)
	*/
	QueryTimeline(timeline time.Time, lp_out_slice interface{}) error
	QueryTimelineUnix(timeline int64, lp_out_slice interface{}) error
	// query the data of a certain time interval and return the data to lp_out_map,
	// typed protobuf.proto
	// ErrorDBFileNotHit
	/*
		var out_map [string][]StoreData // keys is timeline.Format(timekeyformat)
		var out_map [uint32][]StoreData // keys is timeline.Unix()
		var out_map [int64][]StoreData  // keys is timeline.Unix()
		var out_map [uint64][]StoreData // keys is timeline.Unix()

		@example

		db := InitDB()
		beginTimestamp := time.Date(2020, 9, 22, 5, 0, 00, 0, time.Local)
		endTimestamp := time.Date(2020, 9, 22, 5, 2, 00, 0, time.Local)
		map := make(map[string][]types.ProcessInfo)
		db.QueryBetween(beginTimestamp, endTimestamp, &map)
	*/
	QueryBetween(begin time.Time, end time.Time, lp_out_map interface{}) error
	QueryBetweenUnix(begin int64, end int64, lp_out_map interface{}) error

	/* Delete the stored file for the current day of the timeline */
	DeleteStorageFile(timeline time.Time) error
	DeleteStorageFileUnix(timeline int64) error

	/* Get data file storage directory */
	StorageDirectory() string

	/* Determine whether the specified date has expired in the database */
	IsExpired(timeline time.Time, now *time.Time) bool

	/* Destroy database objects */
	Dispose() error
}

func InitDB 

func InitDB(opts ...Option) (SnapsDB, error)

初始化一个数据库 

snapPath := filepath.Join(pkg.AssemblyDir(), "snapsdata/proc")
snapDB, err := snapsdb.InitDB(snapsdb.WithDataPath(snapPath), snapsdb.WithDataRetention(time.Hour*24*14))

type StoreData 

type StoreData protoreflect.ProtoMessage

type StoreFile 

type StoreFile interface {
	// 写入数据
	Write(timestamp int64, data ...StoreData) error
	// query a timeline for data and return to a list
	QueryTimeline(timestamp int64, slice_pointer *reflect.Value, origin_slice *reflect.Value, element_type *reflect.Type) error
	// Query the data of a certain time interval and fill it with map[][]typed
	QueryBetween(begin int64, end int64, map_object reflect.Value, key_type *reflect.Kind, slice_type *reflect.Type, element_type *reflect.Type) error
	// close file
	Close()
	// read file timeline meta information
	ReadMateInfo(timeline int64) (*timelineMateInfo, error)
	/* get file  time base line*/
	TimeBaseline() int64
}

type TagPair added in v1.0.4 

type TagPair map[string]interface{}

value type ：string、int、uint、time、bool

type TagValue added in v1.0.4 

type TagValue interface {
	int
	float64
	string
	bool
}

type ValuePair added in v1.0.4 

type ValuePair map[string]interface{}

value type ：string 、int、uint、float、time、bool、map、struct、slice

Source Files

View all Source files

Directories

Path Synopsis
test
types

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.