jsum

README

JSON Summary

A tool to analyse the structure of JSON from a set of example JSON values.

Rationale

To understand the meaning of a set of JSON values it is often not enough to have e.g. a JSON schema. I wanted a tool to give me a more detailed summary of what is contained in some large files with JSON Lines in it. E.g.:

• Are the values of a number member all integer?
• What is the range of numbers in a number value?
• Does a string value only contain a small restricted set of values, i.e. could it be some sort of enumerated value?
• …etc

The tools I found were focused on JSON Schema, so I rolled my own…

Package

1. On top level this Go package is a library that provides all the things to find the JSON summary. Use it with

   go get git.fractalqb.de/fractalqb/jsum

2. In the cmd/jsum directory you find a simple command line tool to generate summaries. Install it with

   go install git.fractalqb.de/fractalqb/jsum/cmd/jsum@latest

Whats next

• JSUM is currently in a "enough to give me the summary" state, it has a reasonable software design, but it breaks with many software engineering practices, first of all: tests. – This should be improved

• In principle the algorithm would also work in a streaming mode. But the current implementation depends on complete JSON values to be read into a Go interface{}. Quite standard for Go! It does not really hurt for the files I'm currently working with, but… – This could be improved

• There are no tools to control the level of detail. – This cloud be improved

• For more insight it (IMHO) would be very helpful to detect schema patters that occur in different places. I.e. automatically detect common types that are reused in the schema. – Already have some infrastructure for that but its WiP

Documentation

Index

• Constants
• type Any
  • func (*Any) Accepts(v interface{}) bool
  • func (d *Any) Copies() []Deducer
  • func (a *Any) Equal(d Deducer) bool
  • func (a *Any) Example(v interface{}) Deducer
  • func (a *Any) Hash(dh DedupHash) uint64
  • func (d *Any) Nullable() bool
• type Array
  • func (a *Array) Accepts(v interface{}) bool
  • func (d *Array) Copies() []Deducer
  • func (a *Array) Equal(d Deducer) bool
  • func (a *Array) Example(v interface{}) Deducer
  • func (a *Array) Hash(dh DedupHash) uint64
  • func (d *Array) Nullable() bool
• type Boolean
  • func (a *Boolean) Accepts(v interface{}) bool
  • func (d *Boolean) Copies() []Deducer
  • func (s *Boolean) Equal(d Deducer) bool
  • func (a *Boolean) Example(v interface{}) Deducer
  • func (a *Boolean) Hash(dh DedupHash) uint64
  • func (d *Boolean) Nullable() bool
• type Config
• type Deducer
  • func Deduce(cfg *Config, v interface{}) Deducer
• type DedupBool
• type DedupHash
  • func (dh DedupHash) ReusedTypes() (res []Deducer)
• type DedupNumber
• type DedupString
• type Format
• type Invalid
  • func (Invalid) Accepts(_ interface{}) bool
  • func (Invalid) Copies() []Deducer
  • func (Invalid) Equal(_ Deducer) bool
  • func (i Invalid) Example(v interface{}) Deducer
  • func (Invalid) Hash(dh DedupHash) uint64
  • func (Invalid) Nullable() bool
• type JsonType
  • func JsonTypeOf(v interface{}) JsonType
  • func (jt JsonType) Scalar() bool
• type Number
  • func (nr *Number) Accepts(v interface{}) bool
  • func (d *Number) Copies() []Deducer
  • func (nr *Number) Equal(d Deducer) bool
  • func (nr *Number) Example(v interface{}) Deducer
  • func (nr *Number) Hash(dh DedupHash) uint64
  • func (d *Number) Nullable() bool
• type Object
  • func (o *Object) Accepts(v interface{}) bool
  • func (d *Object) Copies() []Deducer
  • func (o *Object) Equal(d Deducer) bool
  • func (o *Object) Example(v interface{}) Deducer
  • func (o *Object) Hash(dh DedupHash) uint64
  • func (d *Object) Nullable() bool
• type String
  • func NewString(cfg *Config, null bool) *String
  • func (a *String) Accepts(v interface{}) bool
  • func (d *String) Copies() []Deducer
  • func (s *String) Equal(d Deducer) bool
  • func (a *String) Example(v interface{}) Deducer
  • func (s *String) Hash(dh DedupHash) uint64
  • func (d *String) Nullable() bool
• type Summary
  • func NewSummary(w io.Writer, cfg *SummaryConfig) *Summary
  • func (s *Summary) Print(scm Deducer) error
• type SummaryConfig
• type Union
  • func (u *Union) Accepts(v interface{}) bool
  • func (d *Union) Copies() []Deducer
  • func (u *Union) Equal(d Deducer) bool
  • func (u *Union) Example(v interface{}) Deducer
  • func (u *Union) Hash(dh DedupHash) uint64
  • func (d *Union) Nullable() bool
• type Unknown
  • func NewUnknown(cfg *Config) *Unknown
  • func (a *Unknown) Accepts(v interface{}) bool
  • func (d *Unknown) Copies() []Deducer
  • func (a *Unknown) Equal(d Deducer) bool
  • func (a *Unknown) Example(v interface{}) Deducer
  • func (a *Unknown) Hash(dh DedupHash) uint64
  • func (d *Unknown) Nullable() bool

Constants

const (
	DateTimeFormat = 1 + iota
)

Types

type Any

type Any struct {
	// contains filtered or unexported fields
}

func (*Any) Accepts

func (*Any) Accepts(v interface{}) bool

func (*Any) Copies

func (d *Any) Copies() []Deducer

func (*Any) Equal

func (a *Any) Equal(d Deducer) bool

func (*Any) Example

func (a *Any) Example(v interface{}) Deducer

func (*Any) Hash

func (a *Any) Hash(dh DedupHash) uint64

func (*Any) Nullable

func (d *Any) Nullable() bool

type Array

type Array struct {
	// contains filtered or unexported fields
}

func (*Array) Accepts

func (a *Array) Accepts(v interface{}) bool

func (*Array) Copies

func (d *Array) Copies() []Deducer

func (*Array) Equal

func (a *Array) Equal(d Deducer) bool

func (*Array) Example

func (a *Array) Example(v interface{}) Deducer

func (*Array) Hash

func (a *Array) Hash(dh DedupHash) uint64

func (*Array) Nullable

func (d *Array) Nullable() bool

type Boolean

type Boolean struct {
	// contains filtered or unexported fields
}

func (*Boolean) Accepts

func (a *Boolean) Accepts(v interface{}) bool

func (*Boolean) Copies

func (d *Boolean) Copies() []Deducer

func (*Boolean) Equal

func (s *Boolean) Equal(d Deducer) bool

func (*Boolean) Example

func (a *Boolean) Example(v interface{}) Deducer

func (*Boolean) Hash

func (a *Boolean) Hash(dh DedupHash) uint64

func (*Boolean) Nullable

func (d *Boolean) Nullable() bool

type Config

type Config struct {
	DedupBool   DedupBool
	DedupNumber DedupNumber
	DedupString DedupString
}

type Deducer

type Deducer interface {
	Accepts(v interface{}) bool
	Example(v interface{}) Deducer
	Nullable() bool
	Hash(dh DedupHash) uint64
	Copies() []Deducer
	Equal(d Deducer) bool
	// contains filtered or unexported methods
}

func Deduce

func Deduce(cfg *Config, v interface{}) Deducer

type DedupBool

type DedupBool uint

const (
	DedupBoolTrue DedupBool = 1 << iota
	DedupBoolFalse
)

type DedupHash

type DedupHash map[uint64][]Deducer

func (DedupHash) ReusedTypes

func (dh DedupHash) ReusedTypes() (res []Deducer)

type DedupNumber

type DedupNumber uint

const (
	DedpuNumberIntFloat DedupNumber = 1 << iota
	DedupNumberFrac
	DedupNumberMin
	DedupNumberMax
	DedupNumberNeg
	DedupNumberPos
)

type DedupString

type DedupString uint

const (
	DedupStringEmpty DedupString = 1 << iota
)

type Format

type Format int

type Invalid

type Invalid struct {
	// contains filtered or unexported fields
}

func (Invalid) Accepts

func (Invalid) Accepts(_ interface{}) bool

func (Invalid) Copies

func (Invalid) Copies() []Deducer

func (Invalid) Equal

func (Invalid) Equal(_ Deducer) bool

func (Invalid) Example

func (i Invalid) Example(v interface{}) Deducer

func (Invalid) Hash

func (Invalid) Hash(dh DedupHash) uint64

func (Invalid) Nullable

func (Invalid) Nullable() bool

type JsonType

type JsonType int

const (
	JsonObject JsonType = iota + 1
	JsonArray
	JsonString
	JsonNumber
	JsonBoolean
)

func JsonTypeOf

func JsonTypeOf(v interface{}) JsonType

func (JsonType) Scalar

func (jt JsonType) Scalar() bool

type Number

type Number struct {
	// contains filtered or unexported fields
}

func (*Number) Accepts

func (nr *Number) Accepts(v interface{}) bool

func (*Number) Copies

func (d *Number) Copies() []Deducer

func (*Number) Equal

func (nr *Number) Equal(d Deducer) bool

func (*Number) Example

func (nr *Number) Example(v interface{}) Deducer

func (*Number) Hash

func (nr *Number) Hash(dh DedupHash) uint64

func (*Number) Nullable

func (d *Number) Nullable() bool

type Object

type Object struct {
	// contains filtered or unexported fields
}

func (*Object) Accepts

func (o *Object) Accepts(v interface{}) bool

func (*Object) Copies

func (d *Object) Copies() []Deducer

func (*Object) Equal

func (o *Object) Equal(d Deducer) bool

func (*Object) Example

func (o *Object) Example(v interface{}) Deducer

func (*Object) Hash

func (o *Object) Hash(dh DedupHash) uint64

func (*Object) Nullable

func (d *Object) Nullable() bool

type String

type String struct {
	// contains filtered or unexported fields
}

func NewString

func NewString(cfg *Config, null bool) *String

func (*String) Accepts

func (a *String) Accepts(v interface{}) bool

func (*String) Copies

func (d *String) Copies() []Deducer

func (*String) Equal

func (s *String) Equal(d Deducer) bool

func (*String) Example

func (a *String) Example(v interface{}) Deducer

func (*String) Hash

func (s *String) Hash(dh DedupHash) uint64

func (*String) Nullable

func (d *String) Nullable() bool

type Summary

type Summary struct {
	SummaryConfig
	// contains filtered or unexported fields
}

func NewSummary

func NewSummary(w io.Writer, cfg *SummaryConfig) *Summary

func (*Summary) Print

func (s *Summary) Print(scm Deducer) error

type SummaryConfig

type SummaryConfig struct {
	TreeStyle *treew.Style
	StringMax int
}

type Union

type Union struct {
	// contains filtered or unexported fields
}

func (*Union) Accepts

func (u *Union) Accepts(v interface{}) bool

func (*Union) Copies

func (d *Union) Copies() []Deducer

func (*Union) Equal

func (u *Union) Equal(d Deducer) bool

func (*Union) Example

func (u *Union) Example(v interface{}) Deducer

func (*Union) Hash

func (u *Union) Hash(dh DedupHash) uint64

func (*Union) Nullable

func (d *Union) Nullable() bool

type Unknown

type Unknown struct {
	// contains filtered or unexported fields
}

func NewUnknown

func NewUnknown(cfg *Config) *Unknown

func (*Unknown) Accepts

func (a *Unknown) Accepts(v interface{}) bool

func (*Unknown) Copies

func (d *Unknown) Copies() []Deducer

func (*Unknown) Equal

func (a *Unknown) Equal(d Deducer) bool

func (*Unknown) Example

func (a *Unknown) Example(v interface{}) Deducer

func (*Unknown) Hash

func (a *Unknown) Hash(dh DedupHash) uint64

func (*Unknown) Nullable

func (d *Unknown) Nullable() bool