Documentation
¶
Index ¶
- Constants
- Variables
- func Add(a, b int) int
- func AlgorithmFromManifestName(filename string) (string, error)
- func BagFileType(name string) string
- func CleanBagName(bagName string) string
- func ContainsControlCharacter(str string) bool
- func ContainsEscapedControl(str string) bool
- func CopyFile(dest, src string) (int64, error)
- func CopyMap[K, V comparable](orig map[K]V) map[K]V
- func DateISO(date time.Time) string
- func DateTimeISO(date time.Time) string
- func DateTimeUS(date time.Time) string
- func DateUS(date time.Time) string
- func Dict(values ...interface{}) (map[string]interface{}, error)
- func DisplayDate(ts time.Time) string
- func EscapeAttr(s string) template.HTMLAttr
- func EscapeHTML(s string) template.HTML
- func EstimatedChunkSize(totalSize float64) uint64
- func ExecCommand(command string, args []string, env []string, stdinData []byte) (stdout, stderr []byte, exitCode int)
- func ExpandTilde(filePath string) (string, error)
- func FileExists(path string) bool
- func FileIconFor(filepath string) template.HTML
- func FindCommonPrefix(paths []string) string
- func GetHashes(algs []string) map[string]hash.Hash
- func GetLogger(level logging.Level) *logging.Logger
- func GetWindowsDrives() []string
- func HasValidExtensionForMimeType(filename, mimeType string) (bool, error)
- func HumanSize(size int64) string
- func IntListContains(list []int, item int) bool
- func IsDirectory(path string) bool
- func IsEmpty(s string) bool
- func IsEmptyStringList(list []string) bool
- func IsListType(obj interface{}) bool
- func IsMapType(obj interface{}) bool
- func LooksLikeHypertextURL(str string) bool
- func LooksLikeManifest(name string) bool
- func LooksLikePayloadFile(name string) bool
- func LooksLikeTagManifest(name string) bool
- func LooksLikeURL(url string) bool
- func LooksLikeUUID(uuid string) bool
- func LooksSafeToDelete(dir string, minLength, minSeparators int) bool
- func MakeTempCSVFileWithValidPaths(t *testing.T, pathToCSVFile string) string
- func Min(x, y int) int
- func Mod(a, b int) bool
- func NewLine() string
- func PathTo(program string) (string, error)
- func PathToTestData() string
- func PathToUnitTestBag(bagName string) string
- func PrintAndExit(message string)
- func ProjectRoot() string
- func ReadFile(filepath string) ([]byte, error)
- func RemoveFromSlice[T any](list []T, index int) []T
- func RunningInCI() bool
- func SetBoolValue(obj interface{}, fieldName string, value bool) error
- func SetFloatValue(obj interface{}, fieldName string, value float64) error
- func SetIntValue(obj interface{}, fieldName string, value int64) error
- func SetStringValue(obj interface{}, fieldName, value string) error
- func SplitAndTrim(s, sep string) []string
- func StrEq(val1, val2 interface{}) bool
- func StrStartsWith(s, prefix string) bool
- func StringIsShellSafe(s string) bool
- func StringListContains(list []string, item string) bool
- func StringListContainsAll(masterList []string, listToCheck []string) bool
- func StringToBool(value string) (bool, error)
- func StripFileExtension(filename string) string
- func StripNonPrintable(str string) string
- func TarPathToBagPath(name string) (string, error)
- func TestsAreRunning() bool
- func ToHumanSize(size, unit int64) string
- func Truncate(value string, length int) string
- func TruncateMiddle(str string, maxLen int) string
- func TruncateStart(str string, maxLen int) string
- func UCFirst(str string) string
- func UnixToISO(ts int64) string
- func YesNo(value bool) string
- func YesOrNo(value bool) string
- type DirectoryStats
- type ExtendedFileInfo
- type NameValuePair
- type NameValuePairList
- type Paths
Constants ¶
const AppName = "DART"
const FileIconGeneric = `<i class="far fa-file"></i>`
const STDIN_ERROR = -10000
Variables ¶
var FileIconMap = map[string]string{}/* 187 elements not displayed */
FileIconMap maps strings to font awesome icons.
var MultipartSuffix = regexp.MustCompile("\\.b\\d+\\.of\\d+$")
MultipartSuffix pattern describes what APTrust's multipart bag suffix looks like. This is for APTrust-specific legacy support.
The tar files that make up multipart bags include a suffix that follows this pattern. For example, after stripping off the .tar suffix, you'll have a name like "my_bag.b04.of12"
var TarSuffix = regexp.MustCompile("\\.tar$")
TarSuffix matches strings that end with .tar
Functions ¶
func BagFileType ¶
func CleanBagName ¶
CleanBagName returns the clean bag name. That's the tar file name minus the tar extension and any ".bagN.ofN" suffix.
func ContainsControlCharacter ¶
ContainsControlCharacter returns true if string str contains a Unicode control character. We use this to test file names, which should not contain control characters.
func ContainsEscapedControl ¶
ContainsEscapedControl returns true if string str contains something that looks like an escaped UTF-8 control character. The Mac OS file system seems to silently escape UTF-8 control characters. That causes problems when we try to copy a file over to another file system that won't accept the control character in a file name. The bag validator looks for file names matching these patterns and rejects them.
func CopyMap ¶
func CopyMap[K, V comparable](orig map[K]V) map[K]V
CopyMap returns a copy of map orig
func DateTimeISO ¶
DateTimeISO returns a date in format "2006-01-02T15:04:05Z"
func DateTimeUS ¶
DateUS returns a date in format "Jan 2, 2006 15:04:05"
func DisplayDate ¶
DisplayDate returns a datetime in human-readable format. This returns an empty string if time is empty.
func EscapeAttr ¶
EscapeAttr escapes an HTML attribute value. This helps avoid the ZgotmplZ problem.
func EscapeHTML ¶
EscapeHTML returns an escaped HTML string. This helps avoid the ZgotmplZ problem.
func EstimatedChunkSize ¶
EstimatedChunkSize returns the size we should use for each chunk in a multipart S3 upload. If we don't tell Minio what chunk size to use, and it doesn't know the size of the total upload, it tries to allocate 5 GB of RAM. This causes some restorations to fail with an out of memory error. https://trello.com/c/1hkP28x1
Param totalSize is the total size of the object to upload. When restoring entire objects, we only know the approximate size, which will be IntellectualObject.FileSize plus one or more payload manifests and tag manifests of unknown size that we'll have to generate on the fly. In practice, we can guesstimate that the total size of a restored object will be about 1.01 - 1.1 times IntellectualObject.FileSize.
Since S3 max upload size is 5 TB with 10k parts, the max this will return is 500MB for part size. Although we could return 5 GB, we don't want to because we can't allocate that much memory inside of memory-limited docker instances.
func ExecCommand ¶
func ExecCommand(command string, args []string, env []string, stdinData []byte) (stdout, stderr []byte, exitCode int)
ExecCommand executes a command and returns the output of STDOUT and STDERR as well as the exit code.
os/exec has some problems when piping stdin. These show up on Travis/Linux, but not Mac. See https://github.com/golang/go/issues/9307
func ExpandTilde ¶
Expands the tilde in a directory path to the current user's home directory. For example, on Linux, ~/data would expand to something like /home/josie/data
func FileExists ¶
Returns true if the file at path exists, false if not.
func FileIconFor ¶
FileIconFor returns a FontAwesome icon for the specified file type, as defined in util.FileIconMap. If the icon map has no entry for this type, this returns util.FileIconGeneric.
func FindCommonPrefix ¶
FindCommonPrefix finds the common prefix in a list of strings. This is used during bagging to trim off extraneous characters from file paths to help determine what their path should be inside the bag.
For example, if we're bagging 100 files that all look like this:
/user/linus/photos/image1.jpg /user/linus/photos/image2.jpg etc.
We don't want the bag's payload directory to look like this:
data/user/linus/photos/image1.jpg data/user/linus/photos/image2.jpg etc.
We want it to look like this:
data/image1.jpg data/image2.jpg etc.
Note that this function has the side effect of sorting the list of paths. While side effects are generally undesirable, this one is OK in the bagging context as it makes the manifests easy to read and tar file structure predictable.
Also note that there may be cases where the list has no common prefix. This will happen, for example, if the user is bagging two directories like "/var/www" and "images".
func GetWindowsDrives ¶
func GetWindowsDrives() []string
GetWindowsDrives returns a list of Windows drives.
func HumanSize ¶
HumanSize returns a number of bytes in a human-readable format. Note that we use base 1024, not base 1000, because AWS uses 1024 to calculate the storage size of the objects we're reporting on.
func IntListContains ¶
IntListContains returns true if list contains item.
func IsEmpty ¶
IsEmpty returns true if string s is empty. Strings that are all whitespace are considered empty.
func IsEmptyStringList ¶
IsEmptyStringList returns true if list contains no items or if all items in list are empty strings.
func IsListType ¶
func IsListType(obj interface{}) bool
IsListType returns true if obj is a slice or array.
func LooksLikeHypertextURL ¶
LooksLikeHypertextURL returns true if str looks like an HTTP or HTTPS URL.
func LooksLikeManifest ¶
func LooksLikePayloadFile ¶
func LooksLikeTagManifest ¶
func LooksLikeURL ¶
LooksLikeURL returns true if url looks like a URL.
func LooksLikeUUID ¶
LooksLikeUUID returns true if uuid looks like a valid UUID.
func LooksSafeToDelete ¶
Returns true if the path specified by dir has at least minLength characters and at least minSeparators path separators. This is for testing paths you want pass into os.RemoveAll(), so you don't wind up deleting "/" or "/etc" or something catastrophic like that.
func MakeTempCSVFileWithValidPaths ¶
Paths in pastbuild_test_batch.csv file are relative. Create a temp file with absolute paths.
func NewLine ¶
func NewLine() string
NewLine returns the correct newline character for the current system, either "\r\n" or "\n".
func PathTo ¶
PathTo returns the path to the specified program. Note that this can be spotty on Windows. We'll need to watch this, log "not found" errors and work from there.
func PathToTestData ¶
func PathToTestData() string
PathToTestData returns the path to the directory containing test data. This is used only in testing.
func PathToUnitTestBag ¶
func PrintAndExit ¶
func PrintAndExit(message string)
PrintAndExit prints a message to STDERR and exits
func RemoveFromSlice ¶
RemoveFromSlice removes the item at index index from slice, returning a new slice.
func RunningInCI ¶
func RunningInCI() bool
RunningInCI returns true when code is running in the Travis CI environment.
func SetBoolValue ¶
SetBoolValue sets the value of obj.fieldName to value. Returns an error if the field does not exist, or is the wrong type, or cannot be set.
func SetFloatValue ¶
SetFloatValue sets the value of obj.fieldName to value. Returns an error if the field does not exist, or is the wrong type, or cannot be set.
func SetIntValue ¶
SetIntValue sets the value of obj.fieldName to value. Returns an error if the field does not exist, or is the wrong type, or cannot be set.
func SetStringValue ¶
SetStringValue sets the value of obj.fieldName to value. Returns an error if the field does not exist, or is the wrong type, or cannot be set.
func SplitAndTrim ¶
SlitAndTrim splits string s on the specified separator, then trims leading and trailing whitespace from each item in the resulting slice. Returns a slice of trimmed strings.
func StrEq ¶
func StrEq(val1, val2 interface{}) bool
StrEq compares the string representation of two values and returns true if they are equal.
func StrStartsWith ¶
StrStartsWith returns true if string s starts with the specified prefix.
func StringIsShellSafe ¶
StringIsShellSafe returns true if string looks safe to pass to shell.
func StringListContains ¶
StringListContains returns true if the list of strings contains item.
func StringListContainsAll ¶
StringListContainsAll returns true if all items in listToCheck are also in masterList. Be sure you pass the params in the right order. Note that this can get expensive if the lists are long.
func StringToBool ¶
StringToBool converts a string to its boolean value, or returns an error if it can't convert the string. In addition to the values supported by strconv.ParseBool, this also supports "yes", "y", "no" and "n". Those strings are case-insensitive.
func StripFileExtension ¶
StripFileExtension returns filename, minus the extension. For example, "my_bag.tar" returns "my_bag".
func StripNonPrintable ¶
StripNonPrintable returns a copy of str with non-printable characters removed.
func TarPathToBagPath ¶
TarPathToBagPath, given the path of a file inside a tarball, returns the path of the file in a bag. The name param generally comes from the Name property of a tar file header. For example, in a tar file called my_bag.tar the paths would translate as follows:
Input -> Output my_bag/bagit.txt -> bagit.txt my_bag/data/file.docx -> data/file.docx my_bag/data/img/photo.jpg -> data/img/photo.jpg
This function assumes (perhaps dangerously) that tarred bags follow the recommdation of pre-1.0 versions of the BagIt spec that say a tarred bag should deserialize to a single top-level directory. This function does not assume that the directory will match the bag name.
func TestsAreRunning ¶
func TestsAreRunning() bool
TestsAreRunning returns true when code is running under "go test"
func ToHumanSize ¶
ToHumanSize converts a raw byte count (size) to a human-friendly representation.
func Truncate ¶
Truncate truncates the value to the given length, appending an ellipses to the end. If value contains HTML tags, they will be stripped because truncating HTML can result in unclosed tags that will ruin the page layout.
func TruncateMiddle ¶
TruncateMiddle trims str to maxLen by removing them from the middle of the string. It adds dots to the middle of the string to indicate text was trimmed.
func TruncateStart ¶
TruncateStart trims str to maxLen by removing them from the start of the string. It adds leading dots to indicate some text was trimmed.
func UCFirst ¶
UCFirst returns string str with the first letter capitalized and all others lower case.
Types ¶
type DirectoryStats ¶
type DirectoryStats struct {
RootIsFile bool
FullPath string
BaseName string
DirCount int
Error string
FileCount int
TotalBytes int64
}
func DirStats ¶
func DirStats(dir string) *DirectoryStats
DirStats returns stats about a directory. This is used in the job files display to show things like total file count, total number of bytes, etc.
func GetDirectoryStats ¶
func GetDirectoryStats(dir string) *DirectoryStats
GetDirectoryStats returns the file count, directory count and total number of bytes found recursively under directory dir.
type ExtendedFileInfo ¶
ExtendedFileInfo adds some data to os.FileInfo including FullPath, Owner() and Group(). The latter two are platform specific. They return zero on Windows and actual ids on posix systems.
func ListDirectory ¶
func ListDirectory(dir string) ([]*ExtendedFileInfo, error)
ListDirectory returns a list of directory contents one level deep. It does not recurse. The list will contain directories first, followed by files.
func ListDirectoryWithSort ¶
func ListDirectoryWithSort(dir string) ([]*ExtendedFileInfo, error)
ListDirectoriesWithSort returns a list of all items in the specified directory, showing folders first, then files. Folders and files are sorted in alpha order, case insensitive.
func NewExtendedFileInfo ¶
func NewExtendedFileInfo(path string, fileInfo os.FileInfo) *ExtendedFileInfo
NewExtendedFileInfo creates a new ExtendedFileInfo object. This takes two params because RecursiveFileList stats the files for us. Otherwise, we'd do it in the constructor.
func RecursiveFileList ¶
func RecursiveFileList(dir string, includeIrregulars bool) ([]*ExtendedFileInfo, error)
RecursiveFileList a list of all items inside of dir. If includeIrregulars is false, this will NOT return links, pipes, devices, or anything else besides regular files and directories.
We generally do want to omit items like symlinks, pipes, etc. when bagging because we cannot bag them.
func (*ExtendedFileInfo) OwnerAndGroup ¶
func (fi *ExtendedFileInfo) OwnerAndGroup() (uid int, gid int)
OwnerAndGrooup returns the file's owner id and group id on posix systems. Returns zero, zero on Windows.
type NameValuePair ¶
NameValuePair contains a name and value. Use this to keep ordered lists of pairs, and lists that contain multiple values for the same named key. Go's maps can't do either of those things.
type NameValuePairList ¶
type NameValuePairList struct {
Items []NameValuePair
}
func NewNameValuePairList ¶
func NewNameValuePairList() *NameValuePairList
func ParseCSV ¶
func ParseCSV(pathToCSVFile string) ([]string, []*NameValuePairList, error)
ParseCSV parses the CSV file at pathToCSVFile and returns the following:
1. A slice of strings containing the entries in the first line of the file. These are assumed to be headers / field names.
2. A slice of url.Values objects in which names are column headers and values are the values parsed from one line of the file. We use url.Values because, unlike a map, it preserves the order of the fields and allows us to have multiple values per key. This is essential as the BagIt spec allows a tag value to be specified multiple times.
3. An error, which will typically be one of: "file does not exist", "file can't be read (permissions)" or "csv parse error".
This will not parse correctly if the first line of the file does not contain headers.
func (*NameValuePairList) Add ¶
func (list *NameValuePairList) Add(name, value string)
func (*NameValuePairList) AllMatching ¶
func (list *NameValuePairList) AllMatching(name string) []NameValuePair
func (*NameValuePairList) FirstMatching ¶
func (list *NameValuePairList) FirstMatching(name string) (NameValuePair, bool)
type Paths ¶
type Paths struct {
Cache string
Config string
DataDir string
Desktop string
Documents string
Downloads string
Music string
Photos string
Videos string
Home string
Public string
LogDir string
TempDir string
Root string
UserMount string
// contains filtered or unexported fields
}
Paths contains paths to common directories used by DART. This was ported from https://github.com/sindresorhus/env-paths so that paths in DART3 will match paths used in prior versions of DART.
func NewPaths ¶
func NewPaths() *Paths
NewPaths returns a Paths struct appropriate to the current operating system. Currently supports only Windows, Mac and Linux.
func (*Paths) DefaultPaths ¶
func (p *Paths) DefaultPaths() ([]*ExtendedFileInfo, error)
DefaultPaths returns a list of ExtendedFileInfo objects describing which directories we should should by default in our file browser.
Source Files