anonymizeindex

Documentation

Index

• Variables

Variables

var RootCmd = &cobra.Command{
	Use:   "anonymize_index",
	Short: "Outputs all items in the index applying some form of anonymization.",
	Long:  "Outputs all items in the index applying some form of anonymization. The anonymous result can be bulk-inserted into another elasticsearch index.",
	RunE: func(cmd *cobra.Command, args []string) error {
		if len(args) == 0 {
			return fmt.Errorf("please specify the URL")
		}
		if anonymizedMap == "" {
			return fmt.Errorf("please specify anonymized_map_path")
		}

		var searchResp *searchResponse
		var err error
		var docsFetched int

		anonymizer, err := newAnonymizer(cont, anonymizedMap, anonFields)
		if err != nil {
			return err
		}

		if cont {
			content, err := ioutil.ReadFile(constFileName)
			if err != nil {
				return err
			}
			searchResp = &searchResponse{ScrollID: string(content)}
		} else {

			http.NewRequest("DELETE", fmt.Sprintf("%s/_search/scroll/_all", strings.TrimRight(args[0], "/")), nil)

			searchResp, err = makeRequest(
				fmt.Sprintf("%s/_search?scroll=%s", strings.TrimRight(args[0], "/"), ctxDuration),
				fmt.Sprintf(`{"from":%d, "size":%d,"query":{"match_all":{}}}`, from, size))
			if err != nil {
				return err
			}
			anonymizer.Anonymize(searchResp.Hits.Hits...)
			if err := printHits(searchResp); err != nil {
				return err
			}
		}

		var unusedScrollID string
		for {
			unusedScrollID = searchResp.ScrollID

			searchResp, err = makeRequest(
				strings.TrimRight(args[0], "/")+"/_search/scroll",
				fmt.Sprintf(`{"scroll": "%s","scroll_id":"%s"}`, ctxDuration, searchResp.ScrollID),
			)
			if err != nil {
				return err
			}

			anonymizer.Anonymize(searchResp.Hits.Hits...)
			if err := printHits(searchResp); err != nil {
				return err
			}
			if len(searchResp.Hits.Hits) < size-from {
				break
			}
			docsFetched += len(searchResp.Hits.Hits)
			if total > 0 && docsFetched >= total {
				break
			}

			if _, err := http.NewRequest("DELETE", fmt.Sprintf("%s/_search/scroll/%s", strings.TrimRight(args[0], "/"), unusedScrollID), nil); err != nil {
				return err
			}
		}

		ioutil.WriteFile(constFileName, []byte(searchResp.ScrollID), 0666)
		if anonymizedMap != "" {
			anonymizer.WriteFieldsMapToFile(anonymizedMap)
		}
		return nil
	},
}

RootCmd is the root of the anonymize command.