hitcounter

Documentation

Overview

hitcounter command uses the search pattern (passed in via STDIN) to count the number of hits of each term inside the passed-in dictionary. The result is an ordered list of pairs term:count (descending on the number of hits), which is written to STDOUT.

Index

• Variables
• type Hit
• type HitsByCount
  • func (a HitsByCount) Len() int
  • func (a HitsByCount) Less(i, j int) bool
  • func (a HitsByCount) Swap(i, j int)

Variables

var RootCmd = &cobra.Command{
	Use:   "counthits",
	Short: "Counts hits passed-in queries.",
	Long:  "Counts hits passed-in queries.",
	RunE: func(cmd *cobra.Command, args []string) error {
		if len(args) == 0 {
			return fmt.Errorf("please set the url argument.")
		}
		url := args[0]
		buff, err := ioutil.ReadAll(os.Stdin)
		if err != nil {
			return err
		}
		query := string(buff)
		if strings.Contains(string(buff), rdictVar) && dict == "" {
			return fmt.Errorf("query defintion uses $RDICT, please set --dictionary_file.")
		}

		clients := make(chan *http.Client, numClients)
		for i := 0; i < numClients; i++ {
			clients <- &http.Client{
				Transport: &http.Transport{
					Dial: (&net.Dialer{
						LocalAddr: &net.TCPAddr{IP: defaultLocalAddr.IP, Zone: defaultLocalAddr.Zone},
						KeepAlive: 3 * timeout,
						Timeout:   timeout,
					}).Dial,
					ResponseHeaderTimeout: timeout,
					MaxIdleConnsPerHost:   defaultConnections,
				},
			}
		}
		errChan := make(chan error)
		var hits HitsByCount
		dictF, err := os.Open(dict)
		if err != nil {
			return err
		}
		defer dictF.Close()
		count := 0
		scanner := bufio.NewScanner(dictF)
		wg := sync.WaitGroup{}
		for ; scanner.Scan(); count++ {
			wg.Add(1)
			go func(term string, count int) {
				defer wg.Done()

				client := <-clients
				defer func() {
					clients <- client
				}()

				query := strings.Replace(query, rdictVar, term, 1)

				ctx, cancel := context.WithTimeout(context.Background(), timeout)
				defer cancel()

				req, err := http.NewRequest("GET", url, strings.NewReader(query))
				if err != nil {

					errChan <- err
					return
				}
				req.WithContext(ctx)

				if debug {
					dReq, _ := httputil.DumpRequest(req, true)
					fmt.Printf("Processing term: %s\n", term)
					fmt.Println(string(dReq))
				}

				resp, err := client.Do(req)
				if err != nil {

					errChan <- err
					return
				}
				defer resp.Body.Close()

				if debug {
					dResp, _ := httputil.DumpResponse(resp, true)
					fmt.Println(string(dResp))
				}

				code := resp.StatusCode
				if resp.StatusCode != http.StatusOK {
					dReq, _ := httputil.DumpRequest(req, true)
					dResp, _ := httputil.DumpResponse(resp, true)
					fmt.Fprintf(os.Stderr, "invalid status code. want:200 got:%d. term:%s lineno:%d req:%s, resp:%s\n", code, term, count+1, string(dReq), string(dResp))
					return
				}
				searchResp := struct {
					Hits struct {
						Total int64 `json:"total"`
					} `json:"hits"`
				}{}
				if err := json.NewDecoder(resp.Body).Decode(&searchResp); err != nil {

					dReq, _ := httputil.DumpRequest(req, true)
					dResp, _ := httputil.DumpResponse(resp, true)
					errChan <- fmt.Errorf("error parsing response %q. term:%s lineno:%d req:%s, resp:%s\n", err, term, count+1, string(dReq), string(dResp))
					return
				}
				hits = append(hits, Hit{Term: term, Count: searchResp.Hits.Total})
			}(scanner.Text(), count)
		}
		if err := scanner.Err(); err != nil {
			return err
		}
		go func() {
			wg.Wait()
			close(errChan)
		}()
		for err := range errChan {
			return err
		}

		sort.Sort(hits)

		writer := bufio.NewWriter(os.Stdout)
		defer writer.Flush()

		out, err := json.MarshalIndent(hits, "", " ")
		if err != nil {
			return err
		}
		fmt.Fprint(writer, string(out))
		fmt.Fprintf(os.Stderr, "sent %d requests.", count)
		return nil
	},
}

Types

type Hit

type Hit struct {
	Term  string
	Count int64
}

type HitsByCount

type HitsByCount []Hit

func (HitsByCount) Len

func (a HitsByCount) Len() int

func (HitsByCount) Less

func (a HitsByCount) Less(i, j int) bool

func (HitsByCount) Swap

func (a HitsByCount) Swap(i, j int)