并发写入 csv：“切片边界超出范围”

Question

我正在尝试从网站列表中提取证书信息并将其写入 csv。我不断遇到同样的错误，但并不总是在同一时间和在不同的域上。我在第 63 行收到错误：err := writer.Write(data)main.analyzeDomains(0xc0000840c0, 0xc0000126c0)        /root/BreakCert/SSLCert/src/main.go:95 +0x5fcreated by main.main        /root/BreakCert/SSLCert/src/main.go:113 +0x1bfpanic: runtime error: slice bounds out of rangegoroutine 35 [running]:bufio.(*Writer).Flush(0xc000024140, 0x400002400, 0x0)        /usr/local/go/src/bufio/bufio.go:590 +0x1c0bufio.(*Writer).WriteByte(0xc000024140, 0xc0000aa92c, 0xc000452500, 0x4d1)        /usr/local/go/src/bufio/bufio.go:645 +0x96bufio.(*Writer).WriteRune(0xc000024140, 0xc00000002c, 0x4d1, 0x4d1, 0x0)        /usr/local/go/src/bufio/bufio.go:657 +0x1aaencoding/csv.(*Writer).Write(0xc0000126c0, 0xc00060a000, 0x5, 0x8, 0x2, 0x1a)        /usr/local/go/src/encoding/csv/writer.go:47 +0x4b8main.storeCertificate(0xc00018cb00, 0xc0000126c0, 0xc000396380, 0x12)        /root/BreakCert/SSLCert/src/main.go:63 +0x3e9main.analyzeDomain(0xc000396380, 0x12, 0xc0000126c0)        /root/BreakCert/SSLCert/src/main.go:88 +0x19dmain.analyzeDomains(0xc0000840c0, 0xc0000126c0)        /root/BreakCert/SSLCert/src/main.go:95 +0x5fcreated by main.main        /root/BreakCert/SSLCert/src/main.go:113 +0x1bfexit status 2并像这样使用cat domains | go run main.go域每行包含一个 url。

Answer 1

这是OP问题的一种解决方案

// echo -e "google.com\ncnn.com\nstackoverflow.com" | go run main.go

package main

import (

  "bufio"

  "crypto/rsa"

  "crypto/tls"

  "crypto/x509"

  "encoding/csv"

  "io"

  "log"

  "net"

  "os"

  "strconv"

  "strings"

  "sync"

  "time"

)

func certToCSV(cert *x509.Certificate, domain string) []string {

  var data []string

  data = append(data, domain[:len(domain)-4])

  var org string

  if len(cert.Issuer.Organization) > 0 {

    org = cert.Issuer.Organization[0]

  }

  data = append(data, org)

  if cert.PublicKey != nil {

    rsaPublicKey := cert.PublicKey.(*rsa.PublicKey)

    data = append(data, rsaPublicKey.N.String())

    data = append(data, strconv.Itoa(rsaPublicKey.E))

    data = append(data, strconv.Itoa(rsaPublicKey.Size()))

  }

  return data

}

func getCerts(d string) ([]*x509.Certificate, error) {

  out := []*x509.Certificate{}

  dialer := net.Dialer{}

  dialer.Timeout = 10 * time.Second

  conn, err := tls.DialWithDialer(&dialer, "tcp", d, &tls.Config{

    InsecureSkipVerify: true,

  })

  if err != nil {

    return out, err

  }

  defer conn.Close()

  for _, cert := range conn.ConnectionState().PeerCertificates {

    if v := cert.PublicKeyAlgorithm.String(); v != "RSA" {

      log.Printf("%q not using RSA algorithm but %q", d, cert.PublicKeyAlgorithm)

      continue

    }

    if len(cert.Issuer.Organization) < 1 {

      log.Printf("%q does not have organization", d)

      continue

    }

    out = append(out, cert)

  }

  return out, err

}

func analyze(dst chan []string, src chan string, errs chan error) {

  for domain := range src {

    certs, err := getCerts(domain)

    if err != nil {

      errs <- err

      continue

    }

    for _, cert := range certs {

      record := certToCSV(cert, domain)

      dst <- record

    }

  }

}

func readCSVFile(dst chan string, fp string) error {

  file, err := os.Create(fp)

  if err != nil {

    return err

  }

  defer file.Close()

  scanner := bufio.NewScanner(os.Stdin)

  for scanner.Scan() {

    line := scanner.Text()

    if !strings.Contains(line, ":") {

      line = line + ":443"

    }

    dst <- line

  }

  return scanner.Err()

}

func readCSV(dst chan string, src io.Reader) error {

  scanner := bufio.NewScanner(src)

  for scanner.Scan() {

    line := scanner.Text()

    if !strings.Contains(line, ":") {

      line = line + ":443"

    }

    dst <- line

  }

  return scanner.Err()

}

func writeCSV(dst io.Writer, src chan []string, errs chan error) {

  w := csv.NewWriter(dst)

  for record := range src {

    if err := w.Write(record); err != nil {

      errs <- err

    }

    w.Flush()

  }

  if err := w.Error(); err != nil {

    errs <- err

  }

}

func main() {

  var wg sync.WaitGroup

  errs := make(chan error)

  src := make(chan string)

  t1 := make(chan []string)

  // synchronize all routines to close errs once

  go func() {

    wg.Wait()

    close(errs)

  }()

  var wg2 sync.WaitGroup

  // analyze multiple domains in //

  for i := 0; i < 4; i++ {

    wg.Add(1)

    wg2.Add(1)

    go func() {

      defer wg.Done()

      defer wg2.Done()

      analyze(t1, src, errs)

    }()

  }

  // synchronize with analyze routines to close t1

  go func() {

    wg2.Wait()

    close(t1)

  }()

  // write the csv file

  wg.Add(1)

  go func() {

    defer wg.Done()

    writeCSV(os.Stdout, t1, errs)

  }()

  // read the csv, fail if an error occurs reading the source

  wg.Add(1)

  go func() {

      defer wg.Done()

      err := readCSV(src, os.Stdin)

      if err != nil {

          log.Fatal(err)

      }

      close(src)

  }()

  // read and print errors, adjust exit code

  var exitCode int

  for err := range errs {

    log.Println(err)

    exitCode = 1

  }

  os.Exit(exitCode)

}