diff --git a/down/bt/metainfo/metainfo_test.go b/down/bt/metainfo/metainfo_test.go deleted file mode 100644 index 042b09e3..00000000 --- a/down/bt/metainfo/metainfo_test.go +++ /dev/null @@ -1,15 +0,0 @@ -package metainfo - -import ( - "fmt" - "testing" -) - -func TestParseFromFile(t *testing.T) { - metaInfo, err := ParseFromFile("../testdata/Game.of.Thrones.S08E05.720p.WEB.H264-MEMENTO.torrent") - if err != nil { - panic(err) - } - fmt.Printf("%+v\n", metaInfo.AnnounceList) - fmt.Printf("%+v\n", metaInfo.Info.Files) -} diff --git a/down/bt/torrent/torrent_test.go b/down/bt/torrent/torrent_test.go deleted file mode 100644 index 409249f1..00000000 --- a/down/bt/torrent/torrent_test.go +++ /dev/null @@ -1,32 +0,0 @@ -package torrent - -import ( - "github.com/monkeyWie/gopeed/down/bt/metainfo" - "github.com/monkeyWie/gopeed/down/bt/peer" - "testing" -) - -func TestTorrent_Download(t *testing.T) { - torrent := buildTorrent() - torrent.Download("e:/testbt/download") -} - -func buildTorrent() *Torrent { - metaInfo, err := metainfo.ParseFromFile("../testdata/Game.of.Thrones.S08E05.720p.WEB.H264-MEMENTO.torrent") - if err != nil { - panic(err) - } - // metaInfo.Announce = "udp://exodus.desync.com:6969/announce" - // metaInfo.Announce = "udp://tracker.openbittorrent.com:80/announce" - // metaInfo.Announce = "udp://tracker.cyberia.is:6969/announce" - // metaInfo.Announce = "udp://9.rarbg.me:2780/announce" - // metaInfo.Announce = "udp://9.rarbg.to:2760/announce" - // metaInfo.Announce = "udp://tracker.leechers-paradise.org:6969" - metaInfo.AnnounceList = [][]string{} - metaInfo.AnnounceList = append(metaInfo.AnnounceList, []string{ - "udp://exodus.desync.com:6969/announce", - "udp://tracker.openbittorrent.com:80/announce", - "udp://tracker.leechers-paradise.org:6969", - }) - return NewTorrent(peer.GenPeerID(), metaInfo) -} diff --git a/go.mod b/go.mod index f1f60be4..294d22c2 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/monkeyWie/gopeed go 1.13 require ( + github.com/RoaringBitmap/roaring v0.4.21 github.com/cenkalti/mse v0.0.0-20140930130441-6ef65f170972 github.com/marksamman/bencode v0.0.0-20150821143521-dc84f26e086e ) diff --git a/go.sum b/go.sum index 4fb0ab47..d334cd5d 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,35 @@ +github.com/RoaringBitmap/roaring v0.4.21 h1:WJ/zIlNX4wQZ9x8Ey33O1UaD9TCTakYsdLFSBcTwH+8= +github.com/RoaringBitmap/roaring v0.4.21/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo= github.com/cenkalti/mse v0.0.0-20140930130441-6ef65f170972 h1:FvAasVrPm/VUO8M0odLDqA9Y7VbALYq3PRAwmWGPw78= github.com/cenkalti/mse v0.0.0-20140930130441-6ef65f170972/go.mod h1:wGf99xKTJ8f/a8cC1inMqFz9cGEzISLLid5F6i5fI/g= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2 h1:Ujru1hufTHVb++eG6OuNDKMxZnGIvF6o/u8q/8h2+I4= +github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= +github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 h1:gclg6gY70GLy3PbkQ1AERPfmLMMagS60DKF78eWwLn8= +github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99 h1:twflg0XRTjwKpxb/jFExr4HGq6on2dEOmnL6FV+fgPw= +github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/marksamman/bencode v0.0.0-20150821143521-dc84f26e086e h1:KMs6SK8iDSR1+ZzOK10L5wGPpWDByyvOe5nrqk51g2U= github.com/marksamman/bencode v0.0.0-20150821143521-dc84f26e086e/go.mod h1:+AHfJo5+69p+fjvMJTmYajNP9rFBHQcaTDFmuXRRATI= +github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae h1:VeRdUYdCw49yizlSbMEn2SZ+gT+3IUKx8BqxyQdz+BY= +github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= +github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ= +github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/tinylib/msgp v1.1.0 h1:9fQd+ICuRIu/ue4vxJZu6/LzxN0HwMds2nq/0cFvxHU= +github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= +github.com/willf/bitset v1.1.10 h1:NotGKqX0KwQ72NUzqrjZq5ipPNDQex9lo3WpaS8L2sc= +github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/main.go b/main.go index 69342c01..31489a77 100644 --- a/main.go +++ b/main.go @@ -2,11 +2,11 @@ package main import ( "fmt" - "github.com/monkeyWie/gopeed/down/bt/metainfo" - "github.com/monkeyWie/gopeed/down/bt/peer" - "github.com/monkeyWie/gopeed/down/bt/torrent" - "github.com/monkeyWie/gopeed/down/bt/tracker" - "github.com/monkeyWie/gopeed/down/http" + "github.com/monkeyWie/gopeed/protocol/bt/metainfo" + "github.com/monkeyWie/gopeed/protocol/bt/peer" + "github.com/monkeyWie/gopeed/protocol/bt/torrent" + "github.com/monkeyWie/gopeed/protocol/bt/tracker" + "github.com/monkeyWie/gopeed/protocol/http" ) func main() { diff --git a/down/bt/client.go b/protocol/bt/client.go similarity index 90% rename from down/bt/client.go rename to protocol/bt/client.go index 0d2d368a..d4336ebd 100644 --- a/down/bt/client.go +++ b/protocol/bt/client.go @@ -2,8 +2,8 @@ package bt import ( "crypto/rand" - "github.com/monkeyWie/gopeed/down/bt/metainfo" - "github.com/monkeyWie/gopeed/down/bt/torrent" + "github.com/monkeyWie/gopeed/protocol/bt/metainfo" + "github.com/monkeyWie/gopeed/protocol/bt/torrent" "net/url" ) diff --git a/down/bt/client_test.go b/protocol/bt/client_test.go similarity index 100% rename from down/bt/client_test.go rename to protocol/bt/client_test.go diff --git a/down/bt/metainfo/metainfo.go b/protocol/bt/metainfo/metainfo.go similarity index 88% rename from down/bt/metainfo/metainfo.go rename to protocol/bt/metainfo/metainfo.go index a8df3308..0c10bae5 100644 --- a/down/bt/metainfo/metainfo.go +++ b/protocol/bt/metainfo/metainfo.go @@ -29,10 +29,12 @@ type MetaInfo struct { type Info struct { Name string `json:"name"` - PieceLength uint64 `json:"piece length"` + PieceLength int `json:"piece length"` Pieces [][20]byte - Length uint64 `json:"length"` - Files []File `json:"files"` + // 单个文件时,length为该文件的长度 + Length uint64 `json:"length"` + // 多个文件 + Files []File `json:"files"` } type File struct { @@ -47,6 +49,12 @@ type FileDetail struct { End int64 } +type PieceMapping struct { + file int + begin int64 + end int64 +} + func ParseFromFile(path string) (*MetaInfo, error) { file, err := os.Open(path) if err != nil { @@ -85,12 +93,12 @@ func ParseFromFile(path string) (*MetaInfo, error) { } // 获取某个分片的大小 -func (metaInfo *MetaInfo) GetPieceSize(index int) uint64 { +func (metaInfo *MetaInfo) GetPieceLength(index int) int { // 是否为最后一个分片 if index == len(metaInfo.Info.Pieces)-1 { - size := metaInfo.totalSize % metaInfo.Info.PieceLength + size := metaInfo.totalSize % uint64(metaInfo.Info.PieceLength) if size > 0 { - return size + return int(size) } } return metaInfo.Info.PieceLength diff --git a/protocol/bt/metainfo/metainfo_test.go b/protocol/bt/metainfo/metainfo_test.go new file mode 100644 index 00000000..10a03465 --- /dev/null +++ b/protocol/bt/metainfo/metainfo_test.go @@ -0,0 +1,29 @@ +package metainfo + +import ( + "encoding/hex" + "fmt" + "github.com/RoaringBitmap/roaring" + "testing" +) + +func TestParseFromFile(t *testing.T) { + // metaInfo, err := ParseFromFile("../testdata/Game.of.Thrones.S08E05.720p.WEB.H264-MEMENTO.torrent") + metaInfo, err := ParseFromFile("../testdata/office.torrent") + if err != nil { + panic(err) + } + fmt.Printf("%+v\n", metaInfo.AnnounceList) + fmt.Printf("%+v\n", metaInfo.Info.Files) + fmt.Printf("%+v\n", metaInfo.Info.PieceLength) + fmt.Printf("%+v\n", hex.EncodeToString(metaInfo.infoHash[:])) +} + +func TestParseFromFile2(t *testing.T) { + bitmap := roaring.NewBitmap() + bitmap.AddMany([]uint32{1, 2, 3, 4, 5, 6, 7, 10, 20, 9, 9, 1}) + fmt.Println(bitmap.Contains(1)) + fmt.Println(bitmap.Contains(2)) + fmt.Println(bitmap.Contains(8)) + fmt.Println(bitmap.GetCardinality()) +} diff --git a/down/bt/peer/handshake.go b/protocol/bt/peer/handshake.go similarity index 100% rename from down/bt/peer/handshake.go rename to protocol/bt/peer/handshake.go diff --git a/down/bt/peer/message/message.go b/protocol/bt/peer/message/message.go similarity index 100% rename from down/bt/peer/message/message.go rename to protocol/bt/peer/message/message.go diff --git a/down/bt/peer/message/msg_bitfield.go b/protocol/bt/peer/message/msg_bitfield.go similarity index 100% rename from down/bt/peer/message/msg_bitfield.go rename to protocol/bt/peer/message/msg_bitfield.go diff --git a/down/bt/peer/message/msg_bitfield_test.go b/protocol/bt/peer/message/msg_bitfield_test.go similarity index 100% rename from down/bt/peer/message/msg_bitfield_test.go rename to protocol/bt/peer/message/msg_bitfield_test.go diff --git a/down/bt/peer/message/msg_choke.go b/protocol/bt/peer/message/msg_choke.go similarity index 100% rename from down/bt/peer/message/msg_choke.go rename to protocol/bt/peer/message/msg_choke.go diff --git a/down/bt/peer/message/msg_interested.go b/protocol/bt/peer/message/msg_interested.go similarity index 100% rename from down/bt/peer/message/msg_interested.go rename to protocol/bt/peer/message/msg_interested.go diff --git a/down/bt/peer/message/msg_keepalive.go b/protocol/bt/peer/message/msg_keepalive.go similarity index 100% rename from down/bt/peer/message/msg_keepalive.go rename to protocol/bt/peer/message/msg_keepalive.go diff --git a/down/bt/peer/message/msg_not_interested.go b/protocol/bt/peer/message/msg_not_interested.go similarity index 100% rename from down/bt/peer/message/msg_not_interested.go rename to protocol/bt/peer/message/msg_not_interested.go diff --git a/down/bt/peer/message/msg_piece.go b/protocol/bt/peer/message/msg_piece.go similarity index 100% rename from down/bt/peer/message/msg_piece.go rename to protocol/bt/peer/message/msg_piece.go diff --git a/down/bt/peer/message/msg_request.go b/protocol/bt/peer/message/msg_request.go similarity index 100% rename from down/bt/peer/message/msg_request.go rename to protocol/bt/peer/message/msg_request.go diff --git a/down/bt/peer/message/msg_unchoke.go b/protocol/bt/peer/message/msg_unchoke.go similarity index 100% rename from down/bt/peer/message/msg_unchoke.go rename to protocol/bt/peer/message/msg_unchoke.go diff --git a/down/bt/peer/peer.go b/protocol/bt/peer/peer.go similarity index 100% rename from down/bt/peer/peer.go rename to protocol/bt/peer/peer.go diff --git a/down/bt/testdata/Game.of.Thrones.S08E05.720p.WEB.H264-MEMENTO.torrent b/protocol/bt/testdata/Game.of.Thrones.S08E05.720p.WEB.H264-MEMENTO.torrent similarity index 100% rename from down/bt/testdata/Game.of.Thrones.S08E05.720p.WEB.H264-MEMENTO.torrent rename to protocol/bt/testdata/Game.of.Thrones.S08E05.720p.WEB.H264-MEMENTO.torrent diff --git a/protocol/bt/testdata/office.torrent b/protocol/bt/testdata/office.torrent new file mode 100644 index 00000000..15768d09 Binary files /dev/null and b/protocol/bt/testdata/office.torrent differ diff --git a/down/bt/torrent/peer_conn.go b/protocol/bt/torrent/peer_conn.go similarity index 83% rename from down/bt/torrent/peer_conn.go rename to protocol/bt/torrent/peer_conn.go index 1af8f463..62b1f1b2 100644 --- a/down/bt/torrent/peer_conn.go +++ b/protocol/bt/torrent/peer_conn.go @@ -7,18 +7,23 @@ import ( "errors" "fmt" "github.com/cenkalti/mse" - "github.com/monkeyWie/gopeed/down/bt/metainfo" - "github.com/monkeyWie/gopeed/down/bt/peer" - "github.com/monkeyWie/gopeed/down/bt/peer/message" + "github.com/monkeyWie/gopeed/protocol/bt/metainfo" + "github.com/monkeyWie/gopeed/protocol/bt/peer" + "github.com/monkeyWie/gopeed/protocol/bt/peer/message" + "path/filepath" + "sync/atomic" + "io" "math" "net" "os" "path" - "sync" "time" ) +// 现在主流客户端的block大小都是16KB +const BlockSize = 2 << 13 + type peerConn struct { torrent *Torrent peer *peer.Peer @@ -32,13 +37,16 @@ type peerConn struct { // peer is interested in this client peerInterested bool - bitfield *message.Bitfield - readyComplete bool - pieceDownloadedCh chan bool + bitfield *message.Bitfield + readyComplete bool + + downloadedCh chan bool + // block个数 + blockCount int + // 已经下载的block个数 + blockDownloadedCount int32 // block下载队列,官方推荐为5 - pieceBlockQueue chan interface{} - // piece上所有的block下载状态 - pieceBlockState sync.Map + blockQueue chan interface{} } func NewPeerConn(torrent *Torrent, peer *peer.Peer) *peerConn { @@ -168,42 +176,49 @@ func (pc *peerConn) ready() error { pc.conn.Close() return err } - pc.pieceBlockQueue = make(chan interface{}, 5) + pc.blockQueue = make(chan interface{}, 5) return nil } // 下载指定piece func (pc *peerConn) downloadPiece(index int) error { - pc.pieceDownloadedCh = make(chan bool) - defer close(pc.pieceDownloadedCh) - pc.pieceBlockState = sync.Map{} - pieceLength := pc.torrent.MetaInfo.GetPieceSize(index) - lastOffset := uint64(0) + pieceLength := pc.torrent.MetaInfo.GetPieceLength(index) + lastOffset := 0 + + pc.blockCount = int(math.Ceil(float64(pieceLength) / BlockSize)) + pc.downloadedCh = make(chan bool) + defer close(pc.downloadedCh) waitDuration := time.Second * 30 timer := time.NewTimer(waitDuration) defer timer.Stop() // 按块下载分片 - for lastOffset < pieceLength { - blockLength := math.Min(2<<13, float64(pieceLength-lastOffset)) + for i := 0; i < pc.blockCount; i++ { + offset := pieceLength * i + var blockLength uint32 + if i == pc.blockCount-1 { + // 最后一个block大小需要计算出来 + blockLength = uint32(pieceLength - offset) + } else { + blockLength = BlockSize + } // 标记block开始下载 - lastOffset32 := uint32(lastOffset) - pc.pieceBlockState.Store(lastOffset32, false) + offset32 := uint32(lastOffset) // block下载排队,并做超时处理 timer.Reset(waitDuration) select { - case pc.pieceBlockQueue <- nil: + case pc.blockQueue <- nil: case <-timer.C: return errors.New("download timeout") } - _, err := pc.conn.Write(message.NewRequest(uint32(index), lastOffset32, uint32(blockLength)).Encode()) + // 发起request,对方会响应piece + _, err := pc.conn.Write(message.NewRequest(uint32(index), offset32, blockLength).Encode()) if err != nil { fmt.Println(err) } - lastOffset += uint64(blockLength) } // 监听是否下载完成 - <-pc.pieceDownloadedCh + <-pc.downloadedCh return nil } @@ -237,7 +252,7 @@ func (pc *peerConn) handleBitfield(buf []byte) { } } -// 处理分片下载响应 +// 处理下载响应,每次接收到响应直接将block写入到对应文件中 func (pc *peerConn) handlePiece(buf []byte) { piece := &message.Piece{} piece.Decode(buf) @@ -246,7 +261,7 @@ func (pc *peerConn) handlePiece(buf []byte) { blockLength := int64(len(piece.Block)) pieceBegin := int64(piece.Index) * int64(info.PieceLength) blockBegin := pieceBegin + int64(piece.Begin) - // 获取对应要的文件 + // 计算block对应要写到的文件偏移 var fileBlocks []fileBlock if len(info.Files) == 0 { // 单文件 @@ -269,7 +284,7 @@ func (pc *peerConn) handlePiece(buf []byte) { // 计算block剩余写入字节数 blockWritable := blockLength - blockSeek fb := fileBlock{ - filepath: f.Path[0], + filepath: filepath.Join(f.Path...), fileSeek: blockFileBegin - f.Begin, blockBegin: blockSeek, } @@ -287,10 +302,10 @@ func (pc *peerConn) handlePiece(buf []byte) { } } } - // fmt.Println(fileBlocks) + // 开始写入文件 for _, f := range fileBlocks { func() { - name := path.Join(pc.torrent.Path, f.filepath) + name := filepath.Join(pc.torrent.Path, f.filepath) file, err := os.OpenFile(name, os.O_RDWR|os.O_CREATE, 0644) if err != nil { panic(err) @@ -298,28 +313,18 @@ func (pc *peerConn) handlePiece(buf []byte) { defer file.Close() _, err = file.WriteAt(piece.Block[f.blockBegin:f.blockEnd], f.fileSeek) if err != nil { - fmt.Println(err) + panic(err) } }() } - pc.pieceBlockState.Store(piece.Begin, true) // 出队 - <-pc.pieceBlockQueue - flag := true - // fmt.Printf("index %d: %v\n", piece.Index, pc.pieceBlockState) - pc.pieceBlockState.Range(func(key, value interface{}) bool { - // 如果有还没下载完的block - if !value.(bool) { - flag = false - return false - } - return true - }) + <-pc.blockQueue + // piece全部下载完 - if flag { + if atomic.AddInt32(&pc.blockDownloadedCount, 1) == int32(pc.blockCount) { // 计算piece对应的文件偏移 sha1 := sha1.New() - pieceLength := pc.torrent.MetaInfo.GetPieceSize(int(piece.Index)) + pieceLength := pc.torrent.MetaInfo.GetPieceLength(int(piece.Index)) writeIndex := getWriteFile(pieceBegin, fds) fileBegin := pieceBegin - fds[writeIndex].Begin for _, fd := range fds[writeIndex:] { @@ -339,7 +344,7 @@ func (pc *peerConn) handlePiece(buf []byte) { panic(err) } } - pieceLength -= uint64(written) + pieceLength -= int(written) }() if pieceLength > 0 { // 继续读下个文件 @@ -357,7 +362,7 @@ func (pc *peerConn) handlePiece(buf []byte) { fmt.Printf("piece %d 校验失败\n", piece.Index) } // piece下载完成 - pc.pieceDownloadedCh <- true + pc.downloadedCh <- true } } diff --git a/down/bt/torrent/peer_conn_test.go b/protocol/bt/torrent/peer_conn_test.go similarity index 84% rename from down/bt/torrent/peer_conn_test.go rename to protocol/bt/torrent/peer_conn_test.go index 09656843..31e41c0e 100644 --- a/down/bt/torrent/peer_conn_test.go +++ b/protocol/bt/torrent/peer_conn_test.go @@ -2,8 +2,9 @@ package torrent import ( "fmt" - "github.com/monkeyWie/gopeed/down/bt/peer" - "github.com/monkeyWie/gopeed/down/bt/tracker" + "github.com/monkeyWie/gopeed/protocol/bt/peer" + "github.com/monkeyWie/gopeed/protocol/bt/tracker" + "os" "sync" "testing" "time" @@ -104,7 +105,6 @@ func getUsablePeerMore() chan *peerConn { if err != nil { panic(err) } - fmt.Printf("Tracker end,peer count:%d\n", len(peers)) ch := make(chan *peerConn) for i := range peers { go peerTest(torrent, &peers[i], ch) @@ -145,3 +145,22 @@ func TestSome(t *testing.T) { fmt.Println("111") } + +func TestSome2(t *testing.T) { + wg := sync.WaitGroup{} + wg.Add(10) + file, err := os.OpenFile("e:/testbt/test.data", os.O_RDWR|os.O_CREATE, 0644) + for i := 0; i < 10; i++ { + go func(i int) { + for j := 0; j < 100; j++ { + if err != nil { + panic(err) + } + _, err = file.WriteAt([]byte{byte(i)}, int64(i*8)) + } + wg.Done() + }(i) + } + wg.Wait() + +} diff --git a/down/bt/torrent/peer_pool.go b/protocol/bt/torrent/peer_pool.go similarity index 96% rename from down/bt/torrent/peer_pool.go rename to protocol/bt/torrent/peer_pool.go index 77dd6231..41e444a1 100644 --- a/down/bt/torrent/peer_pool.go +++ b/protocol/bt/torrent/peer_pool.go @@ -1,7 +1,7 @@ package torrent import ( - "github.com/monkeyWie/gopeed/down/bt/peer" + "github.com/monkeyWie/gopeed/protocol/bt/peer" "sync" ) diff --git a/down/bt/torrent/peer_pool_test.go b/protocol/bt/torrent/peer_pool_test.go similarity index 90% rename from down/bt/torrent/peer_pool_test.go rename to protocol/bt/torrent/peer_pool_test.go index bd35c213..765b4080 100644 --- a/down/bt/torrent/peer_pool_test.go +++ b/protocol/bt/torrent/peer_pool_test.go @@ -2,7 +2,7 @@ package torrent import ( "fmt" - "github.com/monkeyWie/gopeed/down/bt/peer" + "github.com/monkeyWie/gopeed/protocol/bt/peer" "testing" ) diff --git a/down/bt/torrent/piece_state.go b/protocol/bt/torrent/piece_state.go similarity index 100% rename from down/bt/torrent/piece_state.go rename to protocol/bt/torrent/piece_state.go diff --git a/down/bt/torrent/torrent.go b/protocol/bt/torrent/torrent.go similarity index 81% rename from down/bt/torrent/torrent.go rename to protocol/bt/torrent/torrent.go index ae31d32c..6e669348 100644 --- a/down/bt/torrent/torrent.go +++ b/protocol/bt/torrent/torrent.go @@ -2,8 +2,8 @@ package torrent import ( "fmt" - "github.com/monkeyWie/gopeed/down/bt/metainfo" - "github.com/monkeyWie/gopeed/down/bt/tracker" + "github.com/monkeyWie/gopeed/protocol/bt/metainfo" + "github.com/monkeyWie/gopeed/protocol/bt/tracker" "sync" "time" ) @@ -43,7 +43,6 @@ func (t *Torrent) Download(path string) { go func(index int) { peer := t.peerPool.get() if peer == nil { - fmt.Println("no peer") time.Sleep(time.Second * 10) t.PiecesState.setState(index, stateReady) <-taskCh @@ -58,7 +57,12 @@ func (t *Torrent) Download(path string) { fmt.Printf("peer ready:%s,download %d\n", peer.Address(), index) err = pc.downloadPiece(index) if err != nil { - t.PiecesState.setState(index, stateDownloading) + fmt.Println(err) + // 下载失败 + t.PiecesState.setState(index, stateReady) + } else { + // 下载完成 + t.PiecesState.setState(index, stateFinished) } t.peerPool.release(peer) } @@ -73,11 +77,7 @@ func (t *Torrent) fetchPeers() { PeerID: t.PeerID, MetaInfo: t.MetaInfo, } - urls := []string{ - "udp://exodus.desync.com:6969/announce", - "udp://tracker.openbittorrent.com:80/announce", - "udp://tracker.leechers-paradise.org:6969", - } + urls := tracker.MetaInfo.AnnounceList[0] var wg sync.WaitGroup ch := make(chan interface{}) wg.Add(len(urls)) @@ -86,7 +86,7 @@ func (t *Torrent) fetchPeers() { defer wg.Done() peers, err := tracker.DoTracker(url) if err == nil { - fmt.Println("tracker end:" + url) + fmt.Printf("Tracker end,url:%s,count:%d\n", url, len(peers)) ch <- nil t.peerPool.put(peers) } diff --git a/protocol/bt/torrent/torrent_test.go b/protocol/bt/torrent/torrent_test.go new file mode 100644 index 00000000..feb799c9 --- /dev/null +++ b/protocol/bt/torrent/torrent_test.go @@ -0,0 +1,44 @@ +package torrent + +import ( + "github.com/monkeyWie/gopeed/protocol/bt/metainfo" + "github.com/monkeyWie/gopeed/protocol/bt/peer" + "testing" +) + +func TestTorrent_Download(t *testing.T) { + torrent := buildTorrent() + torrent.Download("e:/testbt/download") +} + +func buildTorrent() *Torrent { + // metaInfo, err := metainfo.ParseFromFile("../testdata/Game.of.Thrones.S08E05.720p.WEB.H264-MEMENTO.torrent") + metaInfo, err := metainfo.ParseFromFile("../testdata/office.torrent") + if err != nil { + panic(err) + } + + metaInfo.AnnounceList[0] = append(metaInfo.AnnounceList[0], []string{ + "udp://tracker.coppersurfer.tk:6969/announce", + "udp://tracker.leechers-paradise.org:6969/announce", + "udp://tracker.opentrackr.org:1337/announce", + "udp://p4p.arenabg.com:1337/announce", + "udp://9.rarbg.to:2710/announce", + "udp://9.rarbg.me:2710/announce", + "udp://tracker.internetwarriors.net:1337/announce", + "udp://exodus.desync.com:6969/announce", + "udp://tracker.tiny-vps.com:6969/announce", + "udp://tracker.sbsub.com:2710/announce", + "udp://retracker.lanta-net.ru:2710/announce", + "udp://open.stealth.si:80/announce", + "udp://open.demonii.si:1337/announce", + "udp://tracker.torrent.eu.org:451/announce", + "udp://tracker.moeking.me:6969/announce", + "udp://tracker.cyberia.is:6969/announce", + "udp://denis.stalker.upeer.me:6969/announce", + "udp://ipv4.tracker.harry.lu:80/announce", + "udp://zephir.monocul.us:6969/announce", + "udp://xxxtor.com:2710/announce", + }...) + return NewTorrent(peer.GenPeerID(), metaInfo) +} diff --git a/down/bt/tracker/errors.go b/protocol/bt/tracker/errors.go similarity index 100% rename from down/bt/tracker/errors.go rename to protocol/bt/tracker/errors.go diff --git a/down/bt/tracker/tracker.go b/protocol/bt/tracker/tracker.go similarity index 99% rename from down/bt/tracker/tracker.go rename to protocol/bt/tracker/tracker.go index f652ba35..5dd6f114 100644 --- a/down/bt/tracker/tracker.go +++ b/protocol/bt/tracker/tracker.go @@ -4,8 +4,8 @@ import ( "encoding/binary" "errors" "fmt" - "github.com/monkeyWie/gopeed/down/bt/metainfo" - "github.com/monkeyWie/gopeed/down/bt/peer" + "github.com/monkeyWie/gopeed/protocol/bt/metainfo" + "github.com/monkeyWie/gopeed/protocol/bt/peer" "io" "io/ioutil" "math" diff --git a/down/bt/tracker/tracker_test.go b/protocol/bt/tracker/tracker_test.go similarity index 90% rename from down/bt/tracker/tracker_test.go rename to protocol/bt/tracker/tracker_test.go index 906de0b7..123601b2 100644 --- a/down/bt/tracker/tracker_test.go +++ b/protocol/bt/tracker/tracker_test.go @@ -2,7 +2,7 @@ package tracker import ( "fmt" - "github.com/monkeyWie/gopeed/down/bt/metainfo" + "github.com/monkeyWie/gopeed/protocol/bt/metainfo" "testing" ) diff --git a/down/http/down.go b/protocol/http/down.go similarity index 92% rename from down/http/down.go rename to protocol/http/down.go index 2b405b2a..8dba50aa 100644 --- a/down/http/down.go +++ b/protocol/http/down.go @@ -101,10 +101,7 @@ func Down(request *Request) error { if response.Range { cons := 16 chunkSize := response.Size / int64(cons) - var ( - waitGroup = &sync.WaitGroup{} - fileLock = &sync.Mutex{} - ) + waitGroup := &sync.WaitGroup{} waitGroup.Add(cons) for i := 0; i < cons; i++ { start := int64(i) * chunkSize @@ -112,11 +109,11 @@ func Down(request *Request) error { if i == cons-1 { end = response.Size } - go downChunk(request, file, start, end-1, waitGroup, fileLock) + go downChunk(request, file, start, end-1, waitGroup) } waitGroup.Wait() } else { - downChunk(request, file, 0, response.Size, nil, nil) + downChunk(request, file, 0, response.Size, nil) } return nil } @@ -148,7 +145,7 @@ func BuildHTTPClient() *http.Client { return &http.Client{Jar: jar} } -func downChunk(request *Request, file *os.File, start int64, end int64, waitGroup *sync.WaitGroup, fileLock *sync.Mutex) { +func downChunk(request *Request, file *os.File, start int64, end int64, waitGroup *sync.WaitGroup) { if waitGroup != nil { defer waitGroup.Done() } @@ -163,13 +160,11 @@ func downChunk(request *Request, file *os.File, start int64, end int64, waitGrou } defer httpResponse.Body.Close() buf := make([]byte, 8192) - writeIndex := int64(start) + writeIndex := start for { n, err := httpResponse.Body.Read(buf) if n > 0 { - fileLock.Lock() writeSize, err := file.WriteAt(buf[0:n], writeIndex) - fileLock.Unlock() if err != nil { fmt.Println(err) return diff --git a/down/http/down_test.go b/protocol/http/down_test.go similarity index 99% rename from down/http/down_test.go rename to protocol/http/down_test.go index f644e1c6..30f9b6ed 100644 --- a/down/http/down_test.go +++ b/protocol/http/down_test.go @@ -49,6 +49,7 @@ func TestDown(t *testing.T) { return } downMd5 := fileMd5("favicon.ico") + os.Remove("favicon.ico") if "8de7a6a2e786861013d61b77b2394012" != downMd5 { t.Errorf("error md5= %v", downMd5) return diff --git a/down/http/request.go b/protocol/http/request.go similarity index 100% rename from down/http/request.go rename to protocol/http/request.go diff --git a/down/http/response.go b/protocol/http/response.go similarity index 100% rename from down/http/response.go rename to protocol/http/response.go