metadata

Documentation

Index

• Variables

Variables

var CrawlCommand = &cobra.Command{
	Use:   "crawl",
	Short: "Crawl data manual from specify source[sina/net126]",
	Run: func(cmd *cobra.Command, args []string) {
		if err := loadConfig(); err != nil {
			log.Printf("[F] Load config failure, nest error: %v", err)
			return
		}
		opentrace.OtelDSN = cfg.Otel.DSN
		destroy, err := opentrace.InitTraceProvider()
		if err != nil {
			log.Printf("[F] Init trace provider failure, nest error: %v", err)
			return
		}
		defer destroy()

		ctx, span := opentrace.DefaultTracer().Start(context.Background(), "Manual crawl metadata")
		defer span.End()

		etcd.Endpoints = cfg.Etcd.Endpoints
		client, err := etcd.NewClient()
		if err != nil {
			log.Printf("[F] Create etcd client failure, nest error: %v", err)
			span.RecordError(err)
			return
		}
		defer client.Close()

		resolver.Register(lb.NewBuilder(client))

		if err := crawl(ctx); err != nil {
			log.Printf("[F] Crawl data failure, nest error: %v", err)
			span.RecordError(err)
			return
		}
	},
}

var RootCommand = &cobra.Command{
	Use:   "metadata",
	Short: "Metadata root operation",
	Run: func(cmd *cobra.Command, args []string) {
		cmd.Help()
	},
}

var StoreCommand = &cobra.Command{
	Use:   "store",
	Short: "Store metadata to storage",
	Run: func(cmd *cobra.Command, args []string) {
		begin, err := time.Parse(time.DateTime, fmt.Sprintf("%s 00:00:01", beginDateVar))
		if err != nil {
			log.Printf("[F] Parse begin date failure, nest error: %v, begin: %s", err, beginDateVar)
			return
		}
		end, err := time.Parse(time.DateTime, fmt.Sprintf("%s 23:59:59", endDateVar))
		if err != nil {
			log.Printf("[F] Parse end date failure, nest error: %v, end: %s", err, endDateVar)
			return
		}

		if err := loadConfig(); err != nil {
			log.Printf("[F] Load config failure, nest error: %v", err)
			return
		}

		opentrace.OtelDSN = cfg.Otel.DSN
		destroy, err := opentrace.InitTraceProvider()
		if err != nil {
			log.Printf("[F] Init trace provider failure, nest error: %v", err)
			return
		}
		defer destroy()

		mongodb.DSN = cfg.MongoDB.DSN
		if err := mongodb.Connect(); err != nil {
			log.Printf("[F] Create mongodb client failure, nest error: %v", err)
			return
		}

		ctx, span := opentrace.DefaultTracer().Start(context.Background(), "Manual store metadata")
		defer span.End()

		etcd.Endpoints = cfg.Etcd.Endpoints
		client, err := etcd.NewClient()
		if err != nil {
			span.SetStatus(codes.Error, "New etcd client failure")
			span.RecordError(err)
			log.Printf("[F] Create etcd client failure, nest error: %v", err)
			return
		}
		defer client.Close()
		resolver.Register(lb.NewBuilder(client))

		for begin.Before(end) {
			_, newSpan := opentrace.DefaultTracer().Start(ctx, "store one day")
			var (
				date = begin.Format(time.DateOnly)
				now  = time.Now()
			)
			newSpan.SetAttributes(attribute.String("date", date))

			total, stock, day, week, err := store(ctx, date)
			if err != nil {
				log.Printf("[E] => Store data failure, nest error: %v, date: %v", err, date)
			} else {
				fmt.Printf("[I] => Store data success, date: %v, total: %v, stock: %v, day: %v, week: %v, cost: %v\r\n", date, total, stock, day, week, time.Since(now))
			}
			begin = begin.Add(24 * time.Hour)
			newSpan.End()
		}
	},
}