commands

Documentation

Index

• Variables

Variables

var ConvertCommand = cli.Command{
	Name:      "convert",
	Usage:     "convert an image",
	ArgsUsage: "[flags] <source_ref> <target_ref>...",
	Description: `Convert an image format.

e.g., 'ctr-remote convert --estargz --oci example.com/foo:orig example.com/foo:esgz'

Use '--platform' to define the output platform.
When '--all-platforms' is given all images in a manifest list must be available.
`,
	Flags: []cli.Flag{

		cli.BoolFlag{
			Name:  "estargz",
			Usage: "convert legacy tar(.gz) layers to eStargz for lazy pulling. Should be used in conjunction with '--oci'",
		},
		cli.StringFlag{
			Name:  "estargz-record-in",
			Usage: "Read 'ctr-remote optimize --record-out=<FILE>' record file",
		},
		cli.IntFlag{
			Name:  "estargz-compression-level",
			Usage: "eStargz compression level",
			Value: gzip.BestCompression,
		},
		cli.IntFlag{
			Name:  "estargz-chunk-size",
			Usage: "eStargz chunk size",
			Value: 0,
		},
		cli.IntFlag{
			Name:  "estargz-min-chunk-size",
			Usage: "The minimal number of bytes of data must be written in one gzip stream. Note that this adds a TOC property that old reader doesn't understand.",
			Value: 0,
		},
		cli.BoolFlag{
			Name:  "estargz-external-toc",
			Usage: "Separate TOC JSON into another image (called \"TOC image\"). The name of TOC image is the original + \"-esgztoc\" suffix. Both eStargz and the TOC image should be pushed to the same registry. stargz-snapshotter refers to the TOC image when it pulls the result eStargz image.",
		},
		cli.BoolFlag{
			Name:  "estargz-keep-diff-id",
			Usage: "convert to esgz without changing diffID (cannot be used in conjunction with '--estargz-record-in'. must be specified with '--estargz-external-toc')",
		},

		cli.BoolFlag{
			Name:  "zstdchunked",
			Usage: "use zstd compression instead of gzip (a.k.a zstd:chunked). Must be used in conjunction with '--oci'.",
		},
		cli.StringFlag{
			Name:  "zstdchunked-record-in",
			Usage: "Read 'ctr-remote optimize --record-out=<FILE>' record file",
		},
		cli.IntFlag{
			Name:  "zstdchunked-compression-level",
			Usage: "zstd:chunked compression level",
			Value: 3,
		},
		cli.IntFlag{
			Name:  "zstdchunked-chunk-size",
			Usage: "zstd:chunked chunk size",
			Value: 0,
		},

		cli.BoolFlag{
			Name:  "uncompress",
			Usage: "convert tar.gz layers to uncompressed tar layers",
		},
		cli.BoolFlag{
			Name:  "oci",
			Usage: "convert Docker media types to OCI media types",
		},

		cli.StringSliceFlag{
			Name:  "platform",
			Usage: "Convert content for a specific platform",
			Value: &cli.StringSlice{},
		},
		cli.BoolFlag{
			Name:  "all-platforms",
			Usage: "Convert content for all platforms",
		},
	},
	Action: func(context *cli.Context) error {
		var (
			convertOpts = []converter.Opt{}
		)
		srcRef := context.Args().Get(0)
		targetRef := context.Args().Get(1)
		if srcRef == "" || targetRef == "" {
			return errors.New("src and target image need to be specified")
		}

		var platformMC platforms.MatchComparer
		if context.Bool("all-platforms") {
			platformMC = platforms.All
		} else {
			if pss := context.StringSlice("platform"); len(pss) > 0 {
				var all []ocispec.Platform
				for _, ps := range pss {
					p, err := platforms.Parse(ps)
					if err != nil {
						return fmt.Errorf("invalid platform %q: %w", ps, err)
					}
					all = append(all, p)
				}
				platformMC = platforms.Ordered(all...)
			} else {
				platformMC = platforms.DefaultStrict()
			}
		}
		convertOpts = append(convertOpts, converter.WithPlatform(platformMC))

		var layerConvertFunc converter.ConvertFunc
		var finalize func(ctx gocontext.Context, cs content.Store, ref string, desc *ocispec.Descriptor) (*images.Image, error)
		if context.Bool("estargz") {
			esgzOpts, err := getESGZConvertOpts(context)
			if err != nil {
				return err
			}
			if context.Bool("estargz-external-toc") {
				if !context.Bool("estargz-keep-diff-id") {
					layerConvertFunc, finalize = esgzexternaltocconvert.LayerConvertFunc(esgzOpts, context.Int("estargz-compression-level"))
				} else {
					if context.String("estargz-record-in") != "" {
						return fmt.Errorf("option --estargz-keep-diff-id conflicts with --estargz-record-in")
					}
					layerConvertFunc, finalize = esgzexternaltocconvert.LayerConvertLossLessFunc(esgzexternaltocconvert.LayerConvertLossLessConfig{
						CompressionLevel: context.Int("estargz-compression-level"),
						ChunkSize:        context.Int("estargz-chunk-size"),
						MinChunkSize:     context.Int("estargz-min-chunk-size"),
					})
				}
			} else {
				if context.Bool("estargz-keep-diff-id") {
					return fmt.Errorf("option --estargz-keep-diff-id must be used with --estargz-external-toc")
				}
				layerConvertFunc = estargzconvert.LayerConvertFunc(esgzOpts...)
			}
			if !context.Bool("oci") {
				logrus.Warn("option --estargz should be used in conjunction with --oci")
			}
			if context.Bool("uncompress") {
				return errors.New("option --estargz conflicts with --uncompress")
			}
			if context.Bool("zstdchunked") {
				return errors.New("option --estargz conflicts with --zstdchunked")
			}
		}

		if context.Bool("zstdchunked") {
			esgzOpts, err := getZstdchunkedConvertOpts(context)
			if err != nil {
				return err
			}
			layerConvertFunc = zstdchunkedconvert.LayerConvertFuncWithCompressionLevel(
				zstd.EncoderLevelFromZstd(context.Int("zstdchunked-compression-level")), esgzOpts...)
			if !context.Bool("oci") {
				return errors.New("option --zstdchunked must be used in conjunction with --oci")
			}
			if context.Bool("uncompress") {
				return errors.New("option --zstdchunked conflicts with --uncompress")
			}
		}

		if context.Bool("uncompress") {
			layerConvertFunc = uncompress.LayerConvertFunc
		}

		if layerConvertFunc == nil {
			return errors.New("specify layer converter")
		}
		convertOpts = append(convertOpts, converter.WithLayerConvertFunc(layerConvertFunc))

		if context.Bool("oci") {
			convertOpts = append(convertOpts, converter.WithDockerToOCI(true))
		}

		client, ctx, cancel, err := commands.NewClient(context)
		if err != nil {
			return err
		}
		defer cancel()

		ctx, done, err := client.WithLease(ctx)
		if err != nil {
			return err
		}
		defer done(ctx)

		sigCh := make(chan os.Signal, 1)
		signal.Notify(sigCh, os.Interrupt)
		go func() {

			select {
			case s := <-sigCh:
				logrus.Infof("Got %v", s)
				cancel()
			case <-ctx.Done():
			}
		}()
		newImg, err := converter.Convert(ctx, client, targetRef, srcRef, convertOpts...)
		if err != nil {
			return err
		}
		if finalize != nil {
			newI, err := finalize(ctx, client.ContentStore(), targetRef, &newImg.Target)
			if err != nil {
				return err
			}
			is := client.ImageService()
			_ = is.Delete(ctx, newI.Name)
			finimg, err := is.Create(ctx, *newI)
			if err != nil {
				return err
			}
			fmt.Fprintln(context.App.Writer, "extra image:", finimg.Name)
		}
		fmt.Fprintln(context.App.Writer, newImg.Target.Digest.String())
		return nil
	},
}

ConvertCommand converts an image

var FanotifyCommand = cli.Command{
	Name:   "fanotify",
	Hidden: true,
	Action: func(context *cli.Context) error {
		target := context.Args().Get(0)
		if target == "" {
			return fmt.Errorf("target must be specified")
		}
		return service.Serve(target, os.Stdin, os.Stdout)
	},
}

FanotifyCommand notifies filesystem event under the specified directory.

var GetTOCDigestCommand = cli.Command{
	Name:      "get-toc-digest",
	Usage:     "get the digest of TOC of a layer",
	ArgsUsage: "<layer digest>",
	Flags: []cli.Flag{

		cli.BoolFlag{
			Name:  "zstdchunked",
			Usage: "parse layer as zstd:chunked",
		},

		cli.BoolFlag{
			Name:  "dump-toc",
			Usage: "dump TOC instead of digest. Note that the dumped TOC might be formatted with indents so may have different digest against the original in the layer",
		},
	},
	Action: func(clicontext *cli.Context) error {
		layerDgstStr := clicontext.Args().Get(0)
		if layerDgstStr == "" {
			return errors.New("layer digest need to be specified")
		}

		client, ctx, cancel, err := commands.NewClient(clicontext)
		if err != nil {
			return err
		}
		defer cancel()

		layerDgst, err := digest.Parse(layerDgstStr)
		if err != nil {
			return err
		}
		ra, err := client.ContentStore().ReaderAt(ctx, ocispec.Descriptor{Digest: layerDgst})
		if err != nil {
			return err
		}
		defer ra.Close()

		footerSize := estargz.FooterSize
		if clicontext.Bool("zstdchunked") {
			footerSize = zstdchunked.FooterSize
		}
		footer := make([]byte, footerSize)
		if _, err := ra.ReadAt(footer, ra.Size()-int64(footerSize)); err != nil {
			return fmt.Errorf("error reading footer: %w", err)
		}

		var decompressor estargz.Decompressor
		decompressor = new(estargz.GzipDecompressor)
		if clicontext.Bool("zstdchunked") {
			decompressor = new(zstdchunked.Decompressor)
		}

		_, tocOff, tocSize, err := decompressor.ParseFooter(footer)
		if err != nil {
			return fmt.Errorf("error parsing footer: %w", err)
		}
		if tocSize <= 0 {
			tocSize = ra.Size() - tocOff - int64(footerSize)
		}
		toc, tocDgst, err := decompressor.ParseTOC(io.NewSectionReader(ra, tocOff, tocSize))
		if err != nil {
			return fmt.Errorf("error parsing TOC: %w", err)
		}

		if clicontext.Bool("dump-toc") {
			tocJSON, err := json.MarshalIndent(toc, "", "\t")
			if err != nil {
				return fmt.Errorf("failed to marshal toc: %w", err)
			}
			fmt.Println(string(tocJSON))
			return nil
		}
		fmt.Println(tocDgst.String())
		return nil
	},
}

GetTOCDigestCommand outputs TOC info of a layer

var IPFSPushCommand = cli.Command{
	Name:      "ipfs-push",
	Usage:     "push an image to IPFS (experimental)",
	ArgsUsage: "[flags] <image_ref>",
	Flags: []cli.Flag{

		cli.StringSliceFlag{
			Name:  "platform",
			Usage: "Add content for a specific platform",
			Value: &cli.StringSlice{},
		},
		cli.BoolFlag{
			Name:  "all-platforms",
			Usage: "Add content for all platforms",
		},
		cli.BoolTFlag{
			Name:  "estargz",
			Usage: "Convert the image into eStargz",
		},
	},
	Action: func(context *cli.Context) error {
		srcRef := context.Args().Get(0)
		if srcRef == "" {
			return errors.New("image need to be specified")
		}

		var platformMC platforms.MatchComparer
		if context.Bool("all-platforms") {
			platformMC = platforms.All
		} else {
			if pss := context.StringSlice("platform"); len(pss) > 0 {
				var all []ocispec.Platform
				for _, ps := range pss {
					p, err := platforms.Parse(ps)
					if err != nil {
						return fmt.Errorf("invalid platform %q: %w", ps, err)
					}
					all = append(all, p)
				}
				platformMC = platforms.Ordered(all...)
			} else {
				platformMC = platforms.DefaultStrict()
			}
		}

		client, ctx, cancel, err := commands.NewClient(context)
		if err != nil {
			return err
		}
		defer cancel()

		var layerConvert converter.ConvertFunc
		if context.Bool("estargz") {
			layerConvert = estargzconvert.LayerConvertFunc()
		}
		p, err := ipfs.Push(ctx, client, srcRef, layerConvert, platformMC)
		if err != nil {
			return err
		}
		logrus.WithField("CID", p).Infof("Pushed")
		fmt.Println(p)

		return nil
	},
}

IPFSPushCommand pushes an image to IPFS

var OptimizeCommand = cli.Command{
	Name:      "optimize",
	Usage:     "optimize an image with user-specified workload",
	ArgsUsage: "[flags] <source_ref> <target_ref>...",
	Flags: append([]cli.Flag{
		cli.BoolFlag{
			Name:  "reuse",
			Usage: "reuse eStargz (already optimized) layers without further conversion",
		},
		cli.StringSliceFlag{
			Name:  "platform",
			Usage: "Pull content from a specific platform",
			Value: &cli.StringSlice{},
		},
		cli.BoolFlag{
			Name:  "all-platforms",
			Usage: "targeting all platform of the source image",
		},
		cli.BoolFlag{
			Name:  "wait-on-signal",
			Usage: "ignore context cancel and keep the container running until it receives SIGINT (Ctrl + C) sent manually",
		},
		cli.StringFlag{
			Name:  "wait-on-line",
			Usage: "Substring of a stdout line to be waited. When this string is detected, the container will be killed.",
		},
		cli.BoolFlag{
			Name:  "no-optimize",
			Usage: "convert image without optimization",
		},
		cli.StringFlag{
			Name:  "record-out",
			Usage: "record the monitor log to the specified file",
		},
		cli.BoolFlag{
			Name:  "oci",
			Usage: "convert Docker media types to OCI media types",
		},
		cli.IntFlag{
			Name:  "estargz-compression-level",
			Usage: "eStargz compression level",
			Value: gzip.BestCompression,
		},
		cli.BoolFlag{
			Name:  "estargz-external-toc",
			Usage: "Separate TOC JSON into another image (called \"TOC image\"). The name of TOC image is the original + \"-esgztoc\" suffix. Both eStargz and the TOC image should be pushed to the same registry. stargz-snapshotter refers to the TOC image when it pulls the result eStargz image.",
		},
		cli.IntFlag{
			Name:  "estargz-chunk-size",
			Usage: "eStargz chunk size (not applied to zstd:chunked)",
			Value: 0,
		},
		cli.IntFlag{
			Name:  "estargz-min-chunk-size",
			Usage: "The minimal number of bytes of data must be written in one gzip stream. Note that this adds a TOC property that old reader doesn't understand (not applied to zstd:chunked)",
			Value: 0,
		},
		cli.BoolFlag{
			Name:  "zstdchunked",
			Usage: "use zstd compression instead of gzip (a.k.a zstd:chunked)",
		},
		cli.IntFlag{
			Name:  "zstdchunked-compression-level",
			Usage: "zstd:chunked compression level",
			Value: 3,
		},
	}, samplerFlags...),
	Action: func(clicontext *cli.Context) error {
		convertOpts := []converter.Opt{}
		srcRef := clicontext.Args().Get(0)
		targetRef := clicontext.Args().Get(1)
		if srcRef == "" || targetRef == "" {
			return errors.New("src and target image need to be specified")
		}

		var platformMC platforms.MatchComparer
		if clicontext.Bool("all-platforms") {
			platformMC = platforms.All
		} else {
			if pss := clicontext.StringSlice("platform"); len(pss) > 0 {
				var all []ocispec.Platform
				for _, ps := range pss {
					p, err := platforms.Parse(ps)
					if err != nil {
						return fmt.Errorf("invalid platform %q: %w", ps, err)
					}
					all = append(all, p)
				}
				platformMC = platforms.Ordered(all...)
			} else {
				platformMC = platforms.DefaultStrict()
			}
		}
		convertOpts = append(convertOpts, converter.WithPlatform(platformMC))

		if clicontext.Bool("oci") {
			convertOpts = append(convertOpts, converter.WithDockerToOCI(true))
		} else if clicontext.Bool("zstdchunked") {
			return errors.New("option --zstdchunked must be used in conjunction with --oci")
		}

		client, ctx, cancel, err := commands.NewClient(clicontext)
		if err != nil {
			return err
		}
		defer cancel()

		ctx, done, err := client.WithLease(ctx)
		if err != nil {
			return err
		}
		defer done(ctx)

		recordOut, esgzOptsPerLayer, wrapper, err := analyze(ctx, clicontext, client, srcRef)
		if err != nil {
			return err
		}
		if recordOutFile := clicontext.String("record-out"); recordOutFile != "" {
			if err := writeContentFile(ctx, client, recordOut, recordOutFile); err != nil {
				return fmt.Errorf("failed output record file: %w", err)
			}
		}
		var f converter.ConvertFunc
		var finalize func(ctx context.Context, cs content.Store, ref string, desc *ocispec.Descriptor) (*images.Image, error)
		if clicontext.Bool("zstdchunked") {
			f = zstdchunkedconvert.LayerConvertWithLayerOptsFuncWithCompressionLevel(
				zstd.EncoderLevelFromZstd(clicontext.Int("zstdchunked-compression-level")), esgzOptsPerLayer)
		} else if !clicontext.Bool("estargz-external-toc") {
			f = estargzconvert.LayerConvertWithLayerAndCommonOptsFunc(esgzOptsPerLayer,
				estargz.WithCompressionLevel(clicontext.Int("estargz-compression-level")),
				estargz.WithChunkSize(clicontext.Int("estargz-chunk-size")),
				estargz.WithMinChunkSize(clicontext.Int("estargz-min-chunk-size")))
		} else {
			if clicontext.Bool("reuse") {

				return fmt.Errorf("\"estargz-external-toc\" can't be used with \"reuse\" flag")
			}
			f, finalize = esgzexternaltocconvert.LayerConvertWithLayerAndCommonOptsFunc(esgzOptsPerLayer, []estargz.Option{
				estargz.WithChunkSize(clicontext.Int("estargz-chunk-size")),
				estargz.WithMinChunkSize(clicontext.Int("estargz-min-chunk-size")),
			}, clicontext.Int("estargz-compression-level"))
		}
		if wrapper != nil {
			f = wrapper(f)
		}
		layerConvertFunc := logWrapper(f)

		sigCh := make(chan os.Signal, 1)
		signal.Notify(sigCh, os.Interrupt)
		go func() {

			select {
			case s := <-sigCh:
				logrus.Infof("Got %v", s)
				cancel()
			case <-ctx.Done():
			}
		}()
		convertOpts = append(convertOpts, converter.WithLayerConvertFunc(layerConvertFunc))
		newImg, err := converter.Convert(ctx, client, targetRef, srcRef, convertOpts...)
		if err != nil {
			return err
		}
		if finalize != nil {
			newI, err := finalize(ctx, client.ContentStore(), targetRef, &newImg.Target)
			if err != nil {
				return err
			}
			is := client.ImageService()
			_ = is.Delete(ctx, newI.Name)
			finimg, err := is.Create(ctx, *newI)
			if err != nil {
				return err
			}
			fmt.Fprintln(clicontext.App.Writer, "extra image:", finimg.Name)
		}
		fmt.Fprintln(clicontext.App.Writer, newImg.Target.Digest.String())
		return nil
	},
}

OptimizeCommand converts and optimizes an image

var RpullCommand = cli.Command{
	Name:      "rpull",
	Usage:     "pull an image from a registry levaraging stargz snapshotter",
	ArgsUsage: "[flags] <ref>",
	Description: `Fetch and prepare an image for use in containerd levaraging stargz snapshotter.

After pulling an image, it should be ready to use the same reference in a run
command. 
`,
	Flags: append(append(commands.RegistryFlags, commands.LabelFlag,
		cli.BoolFlag{
			Name:  skipContentVerifyOpt,
			Usage: "Skip content verification for layers contained in this image.",
		},
		cli.BoolFlag{
			Name:  "ipfs",
			Usage: "Pull image from IPFS. Specify an IPFS CID as a reference. (experimental)",
		},
		cli.BoolFlag{
			Name:  "use-containerd-labels",
			Usage: "Use labels defined in containerd project",
		},
	), commands.SnapshotterFlags...),
	Action: func(context *cli.Context) error {
		var (
			ref    = context.Args().First()
			config = &rPullConfig{}
		)
		if ref == "" {
			return fmt.Errorf("please provide an image reference to pull")
		}

		client, ctx, cancel, err := commands.NewClient(context)
		if err != nil {
			return err
		}
		defer cancel()

		ctx, done, err := client.WithLease(ctx)
		if err != nil {
			return err
		}
		defer done(ctx)

		fc, err := content.NewFetchConfig(ctx, context)
		if err != nil {
			return err
		}
		config.FetchConfig = fc
		config.containerdLabels = context.Bool("use-containerd-labels")

		if context.Bool(skipContentVerifyOpt) {
			config.skipVerify = true
		}

		if context.Bool("ipfs") {
			r, err := ipfs.NewResolver(ipfs.ResolverOptions{
				Scheme: "ipfs",
			})
			if err != nil {
				return err
			}
			config.Resolver = r
		}
		config.snapshotter = remoteSnapshotterName
		if sn := context.String("snapshotter"); sn != "" {
			config.snapshotter = sn
		}

		return pull(ctx, client, ref, config)
	},
}

RpullCommand is a subcommand to pull an image from a registry levaraging stargz snapshotter