需要安装两个包:

gopm get -g -v golang.org/x/text

gopm get -g -v golang.org/x/net/html

package fetcher

import ( "bufio" "golang.org/x/net/html/charset" "golang.org/x/text/encoding" "golang.org/x/text/encoding/unicode" "golang.org/x/text/transform" "io/ioutil" "net/http" "src/github.com/gpmgo/gopm/modules/log" )

func Fetch(url string) ([]byte, error) { resp, err := http.Get(url) defer resp.Body.Close() if err != nil { return nil, err }

reader := bufio.NewReader(resp.Body) e := determineEncoding(reader) utf8Reader := transform.NewReader(reader, e.NewDecoder())

body, err := ioutil.ReadAll(utf8Reader) if err != nil { panic(err) }

return body, nil }

func determineEncoding(r *bufio.Reader) encoding.Encoding { b, err := r.Peek(1024) if err != nil { log.Error("get code error") return unicode.UTF8 } e, _, _ := charset.DetermineEncoding(b, "") return e }