mongorestore

package
v0.0.0-...-7526ca3 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jul 23, 2020 License: Apache-2.0 Imports: 26 Imported by: 0

Documentation

Overview

Package mongorestore writes BSON data to a MongoDB instance.

Index

Constants

View Source 
const (
	Users = "users"
	Roles = "roles"
)

Specially treated restore collection types.

View Source 
const (
	ObjcheckOption               = "--objcheck"
	OplogReplayOption            = "--oplogReplay"
	OplogLimitOption             = "--oplogLimit"
	OplogFileOption              = "--oplogFile"
	ArchiveOption                = "--archive" // Value is optional, so must use '=' if specifying one
	RestoreDBUsersAndRolesOption = "--restoreDbUsersAndRoles"
	DirectoryOption              = "--dir"
	GzipOption                   = "--gzip"
)

InputOptions command line argument long names

View Source 
const (
	DropOption                     = "--drop"
	DryRunOption                   = "--dryRun"
	WriteConcernOption             = "--writeConcern"
	NoIndexRestoreOption           = "--noIndexRestore"
	ConvertLegacyIndexesOption     = "--convertLegacyIndexes"
	NoOptionsRestoreOption         = "--noOptionsRestore"
	KeepIndexVersionOption         = "--keepIndexVersion"
	MaintainInsertionOrderOption   = "--maintainInsertionOrder"
	NumParallelCollectionsOption   = "--numParallelCollections"
	NumInsertionWorkersOption      = "--numInsertionWorkersPerCollection"
	StopOnErrorOption              = "--stopOnError"
	BypassDocumentValidationOption = "--bypassDocumentValidation"
	PreserveUUIDOption             = "--preserveUUID"
	TempUsersCollOption            = "--tempUsersColl"
	TempRolesCollOption            = "--tempRolesColl"
	BulkBufferSizeOption           = "--batchSize"
	FixDottedHashedIndexesOption   = "--fixDottedHashIndex"
)

OutputOptions command line argument long names

View Source 
const (
	DBOption                         = "--db"
	CollectionOption                 = "--collection"
	ExcludedCollectionsOption        = "--excludeCollection"
	ExcludedCollectionPrefixesOption = "--excludeCollectionsWithPrefix"
	NSExcludeOption                  = "--nsExclude"
	NSIncludeOption                  = "--nsInclude"
	NSFromOption                     = "--nsFrom"
	NSToOption                       = "--nsTo"
)

NSOptions command line argument long names

Variables

View Source 
var Usage = `` /* 427-byte string literal not displayed */

Usage describes basic usage of mongorestore

Functions

func ParseTimestampFlag 

func ParseTimestampFlag(ts string) (primitive.Timestamp, error)

ParseTimestampFlag takes in a string the form of <time_t>:<ordinal>, where <time_t> is the seconds since the UNIX epoch, and <ordinal> represents a counter of operations in the oplog that occurred in the specified second. It parses this timestamp string and returns a bson.MongoTimestamp type.

func SupportsCollectionUUID 

func SupportsCollectionUUID(sp *db.SessionProvider) (bool, error)

SupportsCollectionUUID was removed from common/db/command.go, so copied to here

Types

type FileType 

type FileType uint

FileType describes the various types of restore documents. 

const (
	UnknownFileType FileType = iota
	BSONFileType
	MetadataFileType
)

File types constants used by mongorestore.

type IndexDocument 

type IndexDocument struct {
	Options                 bson.M `bson:",inline"`
	Key                     bson.D `bson:"key"`
	PartialFilterExpression bson.D `bson:"partialFilterExpression,omitempty"`
}

IndexDocument holds information about a collection's index.

type InputOptions 

type InputOptions struct {
	Objcheck               bool   `long:"objcheck" description:"validate all objects before inserting"`
	OplogReplay            bool   `long:"oplogReplay" description:"replay oplog for point-in-time restore"`
	OplogLimit             string `long:"oplogLimit" value-name:"<seconds>[:ordinal]" description:"only include oplog entries before the provided Timestamp"`
	OplogFile              string `long:"oplogFile" value-name:"<filename>" description:"oplog file to use for replay of oplog"`
	Archive                string `` /* 199-byte string literal not displayed */
	RestoreDBUsersAndRoles bool   `long:"restoreDbUsersAndRoles" description:"restore user and role definitions for the given database"`
	Directory              string `long:"dir" value-name:"<directory-name>" description:"input directory, use '-' for stdin"`
	Gzip                   bool   `long:"gzip" description:"decompress gzipped input"`
}

InputOptions defines the set of options to use in configuring the restore process.

func (*InputOptions) Name 

func (*InputOptions) Name() string

Name returns a human-readable group name for input options.

type Metadata 

type Metadata struct {
	Options        bson.D          `bson:"options,omitempty"`
	Indexes        []IndexDocument `bson:"indexes"`
	UUID           string          `bson:"uuid"`
	CollectionName string          `bson:"collectionName"`
}

Metadata holds information about a collection's options and indexes.

type MongoRestore 

type MongoRestore struct {
	ToolOptions   *options.ToolOptions
	InputOptions  *InputOptions
	OutputOptions *OutputOptions
	NSOptions     *NSOptions

	SessionProvider *db.SessionProvider
	ProgressManager progress.Manager

	TargetDirectory string

	// Skip restoring users and roles, regardless of namespace, when true.
	SkipUsersAndRoles bool

	// Reader to take care of BSON input if not reading from the local filesystem.
	// This is initialized to os.Stdin if unset.
	InputReader io.Reader
	// contains filtered or unexported fields
}

MongoRestore is a container for the user-specified options and internal state used for running mongorestore.

func New 

func New(opts Options) (*MongoRestore, error)

New initializes an instance of MongoRestore according to the provided options.

func (*MongoRestore) ApplyOps 

func (restore *MongoRestore) ApplyOps(session *mongo.Client, entries []interface{}) error

ApplyOps is a wrapper for the applyOps database command, we pass in a session to avoid opening a new connection for a few inserts at a time.

func (*MongoRestore) Close 

func (restore *MongoRestore) Close()

Close ends any connections and cleans up other internal state.

func (*MongoRestore) CollectionExists 

func (restore *MongoRestore) CollectionExists(intent *intents.Intent) (bool, error)

CollectionExists returns true if the given intent's collection exists.

func (*MongoRestore) CreateAllIntents 

func (restore *MongoRestore) CreateAllIntents(dir archive.DirLike) error

CreateAllIntents drills down into a dump folder, creating intents for all of the databases and collections it finds.

func (*MongoRestore) CreateCollection 

func (restore *MongoRestore) CreateCollection(intent *intents.Intent, options bson.D, uuid string) error

CreateCollection creates the collection specified in the intent with the given options.

func (*MongoRestore) CreateIndexes 

func (restore *MongoRestore) CreateIndexes(dbName string, collectionName string, indexes []IndexDocument, hasNonSimpleCollation bool) error

CreateIndexes takes in an intent and an array of index documents and attempts to create them using the createIndexes command. If that command fails, we fall back to individual index creation.

func (*MongoRestore) CreateIntentForCollection 

func (restore *MongoRestore) CreateIntentForCollection(db string, collection string, bsonFile archive.DirLike) error

CreateIntentForCollection builds an intent for the given database and collection name along with a path to a .bson collection file. It searches the file's parent directory for a matching metadata file.

This method is not called by CreateIntentsForDB, it is only used in the case where --db and --collection flags are set.

func (*MongoRestore) CreateIntentForOplog 

func (restore *MongoRestore) CreateIntentForOplog() error

CreateIntentForOplog creates an intent for a file that we want to treat as an oplog.

func (*MongoRestore) CreateIntentsForDB 

func (restore *MongoRestore) CreateIntentsForDB(db string, dir archive.DirLike) (err error)

CreateIntentsForDB drills down into the dir folder, creating intents for all of the collection dump files it finds for the db database.

func (*MongoRestore) CreateStdinIntentForCollection 

func (restore *MongoRestore) CreateStdinIntentForCollection(db string, collection string) error

CreateStdinIntentForCollection builds an intent for the given database and collection name that is to be read from standard input

func (*MongoRestore) DropCollection 

func (restore *MongoRestore) DropCollection(intent *intents.Intent) error

DropCollection drops the intent's collection.

func (*MongoRestore) GetDumpAuthVersion 

func (restore *MongoRestore) GetDumpAuthVersion() (int, error)

GetDumpAuthVersion reads the admin.system.version collection in the dump directory to determine the authentication version of the files in the dump. If that collection is not present in the dump, we try to infer the authentication version based on its absence. Returns the authentication version number and any errors that occur.

func (*MongoRestore) HandleInterrupt 

func (restore *MongoRestore) HandleInterrupt()

func (*MongoRestore) HandleNonTxnOp 

func (restore *MongoRestore) HandleNonTxnOp(oplogCtx *oplogContext, op db.Oplog) error

func (*MongoRestore) HandleTxnOp 

func (restore *MongoRestore) HandleTxnOp(oplogCtx *oplogContext, meta txn.Meta, op db.Oplog) error

func (*MongoRestore) LegacyInsertIndex 

func (restore *MongoRestore) LegacyInsertIndex(dbName string, index IndexDocument) error

LegacyInsertIndex takes in an intent and an index document and attempts to create the index on the "system.indexes" collection.

func (*MongoRestore) LoadIndexesFromBSON 

func (restore *MongoRestore) LoadIndexesFromBSON() error

LoadIndexesFromBSON reads indexes from the index BSON files and caches them in the MongoRestore object.

func (*MongoRestore) MetadataFromJSON 

func (restore *MongoRestore) MetadataFromJSON(jsonBytes []byte) (*Metadata, error)

MetadataFromJSON takes a slice of JSON bytes and unmarshals them into usable collection options and indexes for restoring collections.

func (*MongoRestore) ParseAndValidateOptions 

func (restore *MongoRestore) ParseAndValidateOptions() error

ParseAndValidateOptions returns a non-nil error if user-supplied options are invalid.

func (*MongoRestore) Restore 

func (restore *MongoRestore) Restore() Result

Restore runs the mongorestore program.

func (*MongoRestore) RestoreCollectionToDB 

func (restore *MongoRestore) RestoreCollectionToDB(dbName, colName string,
	bsonSource *db.DecodedBSONSource, file PosReader, fileSize int64) Result

RestoreCollectionToDB pipes the given BSON data into the database. Returns the number of documents restored and any errors that occurred.

func (*MongoRestore) RestoreIntent 

func (restore *MongoRestore) RestoreIntent(intent *intents.Intent) Result

RestoreIntent attempts to restore a given intent into MongoDB.

func (*MongoRestore) RestoreIntents 

func (restore *MongoRestore) RestoreIntents() Result

RestoreIntents iterates through all of the intents stored in the IntentManager, and restores them.

func (*MongoRestore) RestoreOplog 

func (restore *MongoRestore) RestoreOplog() error

RestoreOplog attempts to restore a MongoDB oplog.

func (*MongoRestore) RestoreUsersOrRoles 

func (restore *MongoRestore) RestoreUsersOrRoles(users, roles *intents.Intent) error

RestoreUsersOrRoles accepts a users intent and a roles intent, and restores them via _mergeAuthzCollections. Either or both can be nil. In the latter case nothing is done.

_mergeAuthzCollections is an internal server command implemented specifically for mongorestore. Instead of inserting into the admin.system.{roles, users} collections (which isn't allowed due to some locking policies), we construct temporary collections that are then merged with or replace the existing ones.

The "drop" argument that determines whether the merge replaces the existing users/roles or adds to them.

The "db" argument determines which databases' users and roles are merged. If left blank, it merges users and roles from all databases. When the user restores the admin database, we assume they wish to restore the users and roles for all databases, not just the admin ones, so we leave the "db" field blank in that case.

The "temp{Users,Roles}Collection" arguments determine which temporary collections to merge from, and the presence of either determines which collection to merge to. (e.g. if tempUsersCollection is defined, admin.system.users is merged into).

This command must be run on the "admin" database. Thus, the temporary collections must be on the admin db as well. This command must also be run on the primary.

Example command: 

{
   _mergeAuthzCollections: 1,
   db: "foo",
   tempUsersCollection: "myTempUsers"
   drop: true
   writeConcern: {w: "majority"}
}

func (*MongoRestore) ShouldRestoreUsersAndRoles 

func (restore *MongoRestore) ShouldRestoreUsersAndRoles() bool

ShouldRestoreUsersAndRoles returns true if mongorestore should go through through the process of restoring collections pertaining to authentication.

func (*MongoRestore) TimestampBeforeLimit 

func (restore *MongoRestore) TimestampBeforeLimit(ts primitive.Timestamp) bool

TimestampBeforeLimit returns true if the given timestamp is allowed to be applied to mongorestore's target database.

func (*MongoRestore) UpdateAutoIndexId 

func (restore *MongoRestore) UpdateAutoIndexId(options bson.D)

UpdateAutoIndexId updates {autoIndexId: false} to {autoIndexId: true} if the server version is >= 4.0 and the database is not `local`.

func (*MongoRestore) ValidateAuthVersions 

func (restore *MongoRestore) ValidateAuthVersions() error

ValidateAuthVersions compares the authentication version of the dump files and the authentication version of the target server, and returns an error if the versions are incompatible.

type NSOptions 

type NSOptions struct {
	DB                         string   `short:"d" long:"db" value-name:"<database-name>" description:"database to use when restoring from a BSON file"`
	Collection                 string   `short:"c" long:"collection" value-name:"<collection-name>" description:"collection to use when restoring from a BSON file"`
	ExcludedCollections        []string `` /* 188-byte string literal not displayed */
	ExcludedCollectionPrefixes []string `` /* 226-byte string literal not displayed */
	NSExclude                  []string `long:"nsExclude" value-name:"<namespace-pattern>" description:"exclude matching namespaces"`
	NSInclude                  []string `long:"nsInclude" value-name:"<namespace-pattern>" description:"include matching namespaces"`
	NSFrom                     []string `long:"nsFrom" value-name:"<namespace-pattern>" description:"rename matching namespaces, must have matching nsTo"`
	NSTo                       []string `long:"nsTo" value-name:"<namespace-pattern>" description:"rename matched namespaces, must have matching nsFrom"`
}

NSOptions defines the set of options for configuring involved namespaces

func (*NSOptions) Name 

func (*NSOptions) Name() string

Name returns a human-readable group name for output options.

type Options 

type Options struct {
	*options.ToolOptions
	*InputOptions
	*NSOptions
	*OutputOptions
	TargetDirectory string
}

Options defines the set of all options for configuring mongorestore.

func ParseOptions 

func ParseOptions(rawArgs []string, versionStr, gitCommit string) (Options, error)

ParseOptions reads the command line arguments and converts them into options used to configure a MongoRestore instance

type OutputOptions 

type OutputOptions struct {
	Drop   bool `long:"drop" description:"drop each collection before import"`
	DryRun bool `long:"dryRun" description:"view summary without importing anything. recommended with verbosity"`

	// By default mongorestore uses a write concern of 'majority'.
	WriteConcern             string `` /* 192-byte string literal not displayed */
	NoIndexRestore           bool   `long:"noIndexRestore" description:"don't restore indexes"`
	ConvertLegacyIndexes     bool   `` /* 128-byte string literal not displayed */
	NoOptionsRestore         bool   `long:"noOptionsRestore" description:"don't restore collection options"`
	KeepIndexVersion         bool   `long:"keepIndexVersion" description:"don't update index version"`
	MaintainInsertionOrder   bool   `` /* 300-byte string literal not displayed */
	NumParallelCollections   int    `` /* 127-byte string literal not displayed */
	NumInsertionWorkers      int    `` /* 145-byte string literal not displayed */
	StopOnError              bool   `` /* 412-byte string literal not displayed */
	BypassDocumentValidation bool   `long:"bypassDocumentValidation" description:"bypass document validation"`
	PreserveUUID             bool   `long:"preserveUUID" description:"preserve original collection UUIDs (off by default, requires drop)"`
	TempUsersColl            string `long:"tempUsersColl" default:"tempusers" hidden:"true"`
	TempRolesColl            string `long:"tempRolesColl" default:"temproles" hidden:"true"`
	BulkBufferSize           int    `long:"batchSize" default:"1000" hidden:"true"`
	FixDottedHashedIndexes   bool   `` /* 162-byte string literal not displayed */
}

OutputOptions defines the set of options for restoring dump data.

func (*OutputOptions) Name 

func (*OutputOptions) Name() string

Name returns a human-readable group name for output options.

type PosReader 

type PosReader interface {
	io.ReadCloser
	Pos() int64
}

PosReader is a ReadCloser which maintains the position of what has been read from the Reader.

type Result 

type Result struct {
	Successes int64
	Failures  int64
	Err       error
}

Result encapsulates the outcome of a particular restore attempt.

func NewResultFromBulkResult 

func NewResultFromBulkResult(result *mongo.BulkWriteResult, err error) Result

Source Files

View all Source files

Directories

Path Synopsis
Main package for the mongorestore tool.
 Main package for the mongorestore tool.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.