dataprep

Documentation

Index

• Variables

Variables

var AddPieceCmd = &cli.Command{
	Name:      "add-piece",
	Usage:     "Manually add piece info to a preparation. This is useful for pieces prepared by external tools.",
	Category:  "Piece Management",
	ArgsUsage: "<preparation id|name>",
	Before:    cliutil.CheckNArgs,
	Flags: []cli.Flag{
		&cli.StringFlag{
			Name:     "piece-cid",
			Usage:    "CID of the piece",
			Required: true,
		},
		&cli.StringFlag{
			Name:     "piece-size",
			Usage:    "Size of the piece",
			Required: true,
			Value:    "32GiB",
		},
		&cli.StringFlag{
			Name:  "file-path",
			Usage: "Path to the CAR file, used to determine the size of the file and root CID",
		},
		&cli.StringFlag{
			Name:  "root-cid",
			Usage: "Root CID of the CAR file",
		},
		&cli.Int64Flag{
			Name:  "file-size",
			Usage: "Size of the CAR file, this is required for boost online deal. If not set, it will be determined from the file path if provided.",
		},
	},
	Action: func(c *cli.Context) error {
		db, closer, err := database.OpenFromCLI(c)
		if err != nil {
			return errors.WithStack(err)
		}
		defer closer.Close()

		pieces, err := dataprep.Default.AddPieceHandler(c.Context, db, c.Args().Get(0), dataprep.AddPieceRequest{
			PieceCID:  c.String("piece-cid"),
			PieceSize: c.String("piece-size"),
			FilePath:  c.String("file-path"),
			RootCID:   c.String("root-cid"),
			FileSize:  c.Int64("file-size"),
		})
		if err != nil {
			return errors.WithStack(err)
		}
		cliutil.Print(c, pieces)
		return nil
	},
}

var AttachOutputCmd = &cli.Command{
	Name:      "attach-output",
	Usage:     "Attach a output storage to a preparation",
	ArgsUsage: "<preparation id|name> <storage id|name>",
	Category:  "Preparation Management",
	Before:    cliutil.CheckNArgs,
	Action: func(c *cli.Context) error {
		db, closer, err := database.OpenFromCLI(c)
		if err != nil {
			return errors.WithStack(err)
		}
		defer closer.Close()
		prep, err := dataprep.Default.AddOutputStorageHandler(c.Context, db, c.Args().Get(0), c.Args().Get(1))
		if err != nil {
			return errors.WithStack(err)
		}
		cliutil.Print(c, prep)
		return nil
	},
}

var AttachSourceCmd = &cli.Command{
	Name:      "attach-source",
	Usage:     "Attach a source storage to a preparation",
	ArgsUsage: "<preparation id|name> <storage id|name>",
	Category:  "Preparation Management",
	Before:    cliutil.CheckNArgs,
	Action: func(c *cli.Context) error {
		db, closer, err := database.OpenFromCLI(c)
		if err != nil {
			return errors.WithStack(err)
		}
		defer closer.Close()
		prep, err := dataprep.Default.AddSourceStorageHandler(c.Context, db, c.Args().Get(0), c.Args().Get(1))
		if err != nil {
			return errors.WithStack(err)
		}
		cliutil.Print(c, prep)
		return nil
	},
}

var AttachWalletCmd = &cli.Command{
	Name:      "attach-wallet",
	Usage:     "Attach a wallet to a preparation",
	ArgsUsage: "<preparation id|name> <wallet_id>",
	Category:  "Wallet Management",
	Before:    cliutil.CheckNArgs,
	Action: func(c *cli.Context) error {
		db, closer, err := database.OpenFromCLI(c)
		if err != nil {
			return errors.WithStack(err)
		}
		defer closer.Close()
		prep, err := wallet.Default.AttachHandler(c.Context, db, c.Args().Get(0), c.Args().Get(1))
		if err != nil {
			return errors.WithStack(err)
		}
		cliutil.Print(c, prep)
		return nil
	},
}

var CreateCmd = &cli.Command{
	Name:     "create",
	Usage:    "Create a new preparation",
	Category: "Preparation Management",
	Flags: []cli.Flag{
		&cli.StringFlag{
			Name:        "name",
			Usage:       "The name for the preparation",
			DefaultText: "Auto generated",
		},
		&cli.StringSliceFlag{
			Name:  "source",
			Usage: "The id or name of the source storage to be used for the preparation",
		},
		&cli.StringSliceFlag{
			Name:  "output",
			Usage: "The id or name of the output storage to be used for the preparation",
		},
		&cli.StringSliceFlag{
			Name:     "local-source",
			Category: "Quick creation with local source paths",
			Usage:    "The local source path to be used for the preparation. This is a convenient flag that will create a source storage with the provided path",
		},
		&cli.StringSliceFlag{
			Name:     "local-output",
			Category: "Quick creation with local output paths",
			Usage:    "The local output path to be used for the preparation. This is a convenient flag that will create a output storage with the provided path",
		},
		&cli.StringFlag{
			Name:  "max-size",
			Usage: "The maximum size of a single CAR file",
			Value: "31.5GiB",
		},
		&cli.StringFlag{
			Name:        "piece-size",
			Usage:       "The target piece size of the CAR files used for piece commitment calculation",
			Value:       "",
			DefaultText: "Determined by --max-size",
		},
		&cli.BoolFlag{
			Name:  "delete-after-export",
			Usage: "Whether to delete the source files after export to CAR files",
		},
		&cli.BoolFlag{
			Name:  "no-inline",
			Usage: "Whether to disable inline storage for the preparation. Can save database space but requires at least one output storage.",
		},
		&cli.BoolFlag{
			Name:  "no-dag",
			Usage: "Whether to disable maintaining folder dag structure for the sources. If disabled, DagGen will not be possible and folders will not have an associated CID.",
		},
	},
	Action: func(c *cli.Context) error {
		db, closer, err := database.OpenFromCLI(c)
		if err != nil {
			return errors.WithStack(err)
		}
		defer closer.Close()
		db = db.WithContext(c.Context)
		name := c.String("name")
		if name == "" {
			name = util.RandomName()
		}
		sourceStorages := c.StringSlice("source")
		outputStorages := c.StringSlice("output")
		maxSizeStr := c.String("max-size")
		pieceSizeStr := c.String("piece-size")
		for _, sourcePath := range c.StringSlice("local-source") {
			source, err := createStorageIfNotExist(c.Context, db, sourcePath)
			if err != nil {
				return errors.WithStack(err)
			}
			sourceStorages = append(sourceStorages, source.Name)
		}
		for _, outputPath := range c.StringSlice("local-output") {
			output, err := createStorageIfNotExist(c.Context, db, outputPath)
			if err != nil {
				return errors.WithStack(err)
			}
			outputStorages = append(outputStorages, output.Name)
		}

		prep, err := dataprep.Default.CreatePreparationHandler(c.Context, db, dataprep.CreateRequest{
			SourceStorages:    sourceStorages,
			OutputStorages:    outputStorages,
			MaxSizeStr:        maxSizeStr,
			PieceSizeStr:      pieceSizeStr,
			DeleteAfterExport: c.Bool("delete-after-export"),
			Name:              name,
			NoInline:          c.Bool("no-inline"),
			NoDag:             c.Bool("no-dag"),
		})
		if err != nil {
			return errors.WithStack(err)
		}

		cliutil.Print(c, *prep)
		return nil
	},
}

var DetachOutputCmd = &cli.Command{
	Name:      "detach-output",
	Usage:     "Detach a output storage to a preparation",
	ArgsUsage: "<preparation id|name> <storage id|name>",
	Category:  "Preparation Management",
	Before:    cliutil.CheckNArgs,
	Action: func(c *cli.Context) error {
		db, closer, err := database.OpenFromCLI(c)
		if err != nil {
			return errors.WithStack(err)
		}
		defer closer.Close()
		prep, err := dataprep.Default.RemoveOutputStorageHandler(c.Context, db, c.Args().Get(0), c.Args().Get(1))
		if err != nil {
			return errors.WithStack(err)
		}
		cliutil.Print(c, prep)
		return nil
	},
}

var DetachWalletCmd = &cli.Command{
	Name:      "detach-wallet",
	Usage:     "Detach a wallet to a preparation",
	ArgsUsage: "<preparation id|name> <wallet_id>",
	Category:  "Wallet Management",
	Before:    cliutil.CheckNArgs,
	Action: func(c *cli.Context) error {
		db, closer, err := database.OpenFromCLI(c)
		if err != nil {
			return errors.WithStack(err)
		}
		defer closer.Close()
		prep, err := wallet.Default.DetachHandler(c.Context, db, c.Args().Get(0), c.Args().Get(1))
		if err != nil {
			return errors.WithStack(err)
		}
		cliutil.Print(c, prep)
		return nil
	},
}

var ExploreCmd = &cli.Command{
	Name:      "explore",
	Usage:     "Explore prepared source by path",
	ArgsUsage: "<preparation id|name> <storage id|name> [path]",
	Category:  "Preparation Management",
	Before:    cliutil.CheckNArgs,
	Action: func(c *cli.Context) error {
		db, closer, err := database.OpenFromCLI(c)
		if err != nil {
			return errors.WithStack(err)
		}
		defer closer.Close()

		entries, err := dataprep.Default.ExploreHandler(c.Context, db, c.Args().Get(0), c.Args().Get(1), c.Args().Get(2))
		if err != nil {
			return errors.WithStack(err)
		}
		cliutil.Print(c, entries)
		return nil
	},
}

var ListCmd = &cli.Command{
	Name:     "list",
	Usage:    "List all preparations",
	Category: "Preparation Management",
	Action: func(c *cli.Context) error {
		db, closer, err := database.OpenFromCLI(c)
		if err != nil {
			return errors.WithStack(err)
		}
		defer closer.Close()
		preps, err := dataprep.Default.ListHandler(c.Context, db)
		if err != nil {
			return errors.WithStack(err)
		}
		cliutil.Print(c, preps)
		return nil
	},
}

var ListPiecesCmd = &cli.Command{
	Name:      "list-pieces",
	Usage:     "List all generated pieces for a preparation",
	Category:  "Piece Management",
	ArgsUsage: "<preparation id|name>",
	Before:    cliutil.CheckNArgs,
	Action: func(c *cli.Context) error {
		db, closer, err := database.OpenFromCLI(c)
		if err != nil {
			return errors.WithStack(err)
		}
		defer closer.Close()

		pieces, err := dataprep.Default.ListPiecesHandler(c.Context, db, c.Args().Get(0))
		if err != nil {
			return errors.WithStack(err)
		}
		cliutil.Print(c, pieces)
		return nil
	},
}

var ListWalletsCmd = &cli.Command{
	Name:      "list-wallets",
	Usage:     "List attached wallets with a preparation",
	ArgsUsage: "<preparation id|name>",
	Category:  "Wallet Management",
	Before:    cliutil.CheckNArgs,
	Action: func(c *cli.Context) error {
		db, closer, err := database.OpenFromCLI(c)
		if err != nil {
			return errors.WithStack(err)
		}
		defer closer.Close()
		prep, err := wallet.Default.ListAttachedHandler(c.Context, db, c.Args().Get(0))
		if err != nil {
			return errors.WithStack(err)
		}
		cliutil.Print(c, prep)
		return nil
	},
}

var PauseDagGenCmd = &cli.Command{
	Name:      "pause-daggen",
	Usage:     "Pause a DAG generation job",
	Category:  "Job Management",
	ArgsUsage: "<preparation_id> <storage_name>",
	Before:    cliutil.CheckNArgs,
	Action: func(c *cli.Context) error {
		db, closer, err := database.OpenFromCLI(c)
		if err != nil {
			return errors.WithStack(err)
		}
		defer closer.Close()
		job, err := job.Default.PauseDagGenHandler(c.Context, db, c.Args().Get(0), c.Args().Get(1))
		if err != nil {
			return errors.WithStack(err)
		}
		cliutil.Print(c, job)
		return nil
	},
}

var PausePackCmd = &cli.Command{
	Name:      "pause-pack",
	Usage:     "Pause all pack jobs or a specific one",
	Category:  "Job Management",
	ArgsUsage: "<preparation id|name> <storage id|name> [job_id]",
	Before:    cliutil.CheckNArgs,
	Action: func(c *cli.Context) error {
		db, closer, err := database.OpenFromCLI(c)
		if err != nil {
			return errors.WithStack(err)
		}
		defer closer.Close()
		var jobID int64
		if c.Args().Get(2) != "" {
			jobID, err = strconv.ParseInt(c.Args().Get(2), 10, 64)
			if err != nil {
				return errors.Wrapf(err, "invalid job ID '%s'", c.Args().Get(2))
			}
		}
		job, err := job.Default.PausePackHandler(c.Context, db, c.Args().Get(0), c.Args().Get(1), jobID)
		if err != nil {
			return errors.WithStack(err)
		}
		cliutil.Print(c, job)
		return nil
	},
}

var PauseScanCmd = &cli.Command{
	Name:      "pause-scan",
	Usage:     "Pause a scanning job",
	Category:  "Job Management",
	ArgsUsage: "<preparation id|name> <storage id|name>",
	Before:    cliutil.CheckNArgs,
	Action: func(c *cli.Context) error {
		db, closer, err := database.OpenFromCLI(c)
		if err != nil {
			return errors.WithStack(err)
		}
		defer closer.Close()
		job, err := job.Default.PauseScanHandler(c.Context, db, c.Args().Get(0), c.Args().Get(1))
		if err != nil {
			return errors.WithStack(err)
		}
		cliutil.Print(c, job)
		return nil
	},
}

var RemoveCmd = &cli.Command{
	Name:  "remove",
	Usage: "Remove a preparation",
	Description: `This will remove all relevant information, including:
  * All related jobs
  * All related piece info
  * Mapping used for Inline Preparation
  * All File and Directory data and CIDs
  * All Schedules
This will not remove
  * All deals ever made`,
	ArgsUsage: "<name|id>",
	Before:    cliutil.CheckNArgs,
	Flags: []cli.Flag{
		&cli.BoolFlag{
			Name:  "cars",
			Usage: "Also remove prepared CAR files",
		},
	},
	Action: func(c *cli.Context) error {
		db, closer, err := database.OpenFromCLI(c)
		if err != nil {
			return errors.WithStack(err)
		}
		defer closer.Close()

		removeCars := c.Bool("cars")

		err = dataprep.Default.RemovePreparationHandler(
			c.Context, db, c.Args().Get(0),
			dataprep.RemoveRequest{
				RemoveCars: removeCars,
			})

		return errors.WithStack(err)
	},
}

var RenameCmd = &cli.Command{
	Name:      "rename",
	Usage:     "Rename a preparation",
	ArgsUsage: "<name|id> <new_name>",
	Before:    cliutil.CheckNArgs,
	Action: func(c *cli.Context) error {
		db, closer, err := database.OpenFromCLI(c)
		if err != nil {
			return errors.WithStack(err)
		}
		defer closer.Close()

		preparation, err := dataprep.Default.RenamePreparationHandler(c.Context, db, c.Args().Get(0), dataprep.RenameRequest{Name: c.Args().Get(1)})
		if err != nil {
			return errors.WithStack(err)
		}
		cliutil.Print(c, preparation)
		return nil
	},
}

var StartDagGenCmd = &cli.Command{
	Name:      "start-daggen",
	Usage:     "Start a DAG generation that creates a snapshot of all folder structures",
	Category:  "Job Management",
	ArgsUsage: "<preparation id|name> <storage id|name>",
	Before:    cliutil.CheckNArgs,
	Action: func(c *cli.Context) error {
		db, closer, err := database.OpenFromCLI(c)
		if err != nil {
			return errors.WithStack(err)
		}
		defer closer.Close()
		job, err := job.Default.StartDagGenHandler(c.Context, db, c.Args().Get(0), c.Args().Get(1))
		if err != nil {
			return errors.WithStack(err)
		}
		cliutil.Print(c, job)
		return nil
	},
}

var StartPackCmd = &cli.Command{
	Name:      "start-pack",
	Usage:     "Start / Restart all pack jobs or a specific one",
	Category:  "Job Management",
	ArgsUsage: "<preparation id|name> <storage id|name> [job_id]",
	Before:    cliutil.CheckNArgs,
	Action: func(c *cli.Context) error {
		db, closer, err := database.OpenFromCLI(c)
		if err != nil {
			return errors.WithStack(err)
		}
		defer closer.Close()
		var jobID int64
		if c.Args().Get(2) != "" {
			jobID, err = strconv.ParseInt(c.Args().Get(2), 10, 64)
			if err != nil {
				return errors.Wrapf(err, "invalid job ID '%s'", c.Args().Get(2))
			}
		}
		job, err := job.Default.StartPackHandler(c.Context, db, c.Args().Get(0), c.Args().Get(1), jobID)
		if err != nil {
			return errors.WithStack(err)
		}
		cliutil.Print(c, job)
		return nil
	},
}

var StartScanCmd = &cli.Command{
	Name:      "start-scan",
	Usage:     "Start scanning of the source storage",
	Category:  "Job Management",
	ArgsUsage: "<preparation id|name> <storage id|name>",
	Before:    cliutil.CheckNArgs,
	Action: func(c *cli.Context) error {
		db, closer, err := database.OpenFromCLI(c)
		if err != nil {
			return errors.WithStack(err)
		}
		defer closer.Close()
		job, err := job.Default.StartScanHandler(c.Context, db, c.Args().Get(0), c.Args().Get(1))
		if err != nil {
			return errors.WithStack(err)
		}
		cliutil.Print(c, job)
		return nil
	},
}

var StatusCmd = &cli.Command{
	Name:      "status",
	Usage:     "Get the preparation job status of a preparation",
	Category:  "Job Management",
	ArgsUsage: "<preparation id|name>",
	Before:    cliutil.CheckNArgs,
	Action: func(c *cli.Context) error {
		db, closer, err := database.OpenFromCLI(c)
		if err != nil {
			return errors.WithStack(err)
		}
		defer closer.Close()

		status, err := job.Default.GetStatusHandler(c.Context, db, c.Args().Get(0))
		if err != nil {
			return errors.WithStack(err)
		}
		cliutil.Print(c, status)
		return nil
	},
}