包名重构

This commit is contained in:
liwei committed 2020-02-25 14:22:17 +08:00
1 parent fb45066228
commit 8298051c81
37 files changed
+220 -134

No files matched your search

+408
View File
@@ -0,0 +1,408 @@
package torrent
import (
"bufio"
"crypto/sha1"
"encoding/binary"
"errors"
"fmt"
"github.com/cenkalti/mse"
"github.com/monkeyWie/gopeed/protocol/bt/metainfo"
"github.com/monkeyWie/gopeed/protocol/bt/peer"
"github.com/monkeyWie/gopeed/protocol/bt/peer/message"
"path/filepath"
"sync/atomic"
"io"
"math"
"net"
"os"
"path"
"time"
)
// 现在主流客户端的block大小都是16KB
const BlockSize = 2 << 13
type peerConn struct {
torrent *Torrent
peer *peer.Peer
conn net.Conn
// this client is choking the peer
amChoking bool
// this client is interested in the peer
amInterested bool
// peer is choking this client
peerChoking bool
// peer is interested in this client
peerInterested bool
bitfield *message.Bitfield
readyComplete bool
downloadedCh chan bool
// block个数
blockCount int
// 已经下载的block个数
blockDownloadedCount int32
// block下载队列,官方推荐为5
blockQueue chan interface{}
}
func NewPeerConn(torrent *Torrent, peer *peer.Peer) *peerConn {
return &peerConn{
torrent: torrent,
peer: peer,
}
}
// 使用MSE加密来避免运营商对bt流量的封锁,基本上现在市面上BT客户端都默认开启了,不用MSE的话很多Peer拒绝连接
// see http://wiki.vuze.com/w/Message_Stream_Encryption
func (pc *peerConn) dialMse() error {
conn, err := net.DialTimeout("tcp", pc.peer.Address(), time.Second*time.Duration(30))
if err != nil {
return err
}
mseConn := mse.WrapConn(conn)
infoHash := pc.torrent.MetaInfo.GetInfoHash()
_, err = mseConn.HandshakeOutgoing(infoHash[:], mse.PlainText, nil)
if err != nil {
mseConn.Close()
return err
}
pc.conn = mseConn
return nil
}
// Handshake of Peer wire protocol
// see https://wiki.theory.org/index.php/BitTorrentSpecification#Handshake
func (pc *peerConn) handshake() (*peer.Handshake, error) {
handshakeRes, err := func() (*peer.Handshake, error) {
handshakeReq := peer.NewHandshake([8]byte{}, pc.torrent.MetaInfo.GetInfoHash(), pc.torrent.PeerID)
_, err := pc.conn.Write(handshakeReq.Encode())
if err != nil {
return nil, err
}
var read [68]byte
_, err = io.ReadFull(pc.conn, read[:])
if err != nil {
return nil, err
}
handshakeRes := &peer.Handshake{}
err = handshakeRes.Decode(read[:])
if err != nil {
return nil, err
}
// InfoHash不匹配
if handshakeRes.InfoHash != handshakeReq.InfoHash {
return nil, fmt.Errorf("info_hash not currently serving")
}
return handshakeRes, nil
}()
if err != nil {
pc.conn.Close()
return nil, err
}
// init state
pc.amChoking = true
pc.amInterested = false
pc.peerChoking = true
pc.peerInterested = false
return handshakeRes, nil
}
// 准备下载
func (pc *peerConn) ready() error {
if err := pc.dialMse(); err != nil {
return err
}
if _, err := pc.handshake(); err != nil {
return err
}
pc.readyComplete = false
readyCh := make(chan bool)
defer close(readyCh)
go func() {
scanner := bufio.NewScanner(pc.conn)
scanner.Split(message.SplitMessage)
for scanner.Scan() {
buf := scanner.Bytes()
length := binary.BigEndian.Uint32(buf[:4])
if length == 0 {
// keepalive
} else {
messageID := message.ID(buf[4])
switch messageID {
case message.IdChoke:
break
case message.IdUnchoke:
pc.handleUnchoke(readyCh)
break
case message.IdInterested:
break
case message.IdNotinterested:
break
case message.IdHave:
break
case message.IdBitfield:
pc.handleBitfield(buf)
break
case message.IdRequest:
break
case message.IdPiece:
pc.handlePiece(buf)
break
case message.IdCancel:
break
}
}
}
}()
err := func() error {
select {
case status := <-readyCh:
if status {
return nil
} else {
return errors.New("ready fail")
}
case <-time.After(time.Second * 30):
// 30秒之后超时
return errors.New("ready time out")
}
}()
pc.readyComplete = true
if err != nil {
pc.conn.Close()
return err
}
pc.blockQueue = make(chan interface{}, 5)
return nil
}
// 下载指定piece
func (pc *peerConn) downloadPiece(index int) error {
pieceLength := pc.torrent.MetaInfo.GetPieceLength(index)
lastOffset := 0
pc.blockCount = int(math.Ceil(float64(pieceLength) / BlockSize))
pc.downloadedCh = make(chan bool)
defer close(pc.downloadedCh)
waitDuration := time.Second * 30
timer := time.NewTimer(waitDuration)
defer timer.Stop()
// 按块下载分片
for i := 0; i < pc.blockCount; i++ {
offset := pieceLength * i
var blockLength uint32
if i == pc.blockCount-1 {
// 最后一个block大小需要计算出来
blockLength = uint32(pieceLength - offset)
} else {
blockLength = BlockSize
}
// 标记block开始下载
offset32 := uint32(lastOffset)
// block下载排队,并做超时处理
timer.Reset(waitDuration)
select {
case pc.blockQueue <- nil:
case <-timer.C:
return errors.New("download timeout")
}
// 发起request,对方会响应piece
_, err := pc.conn.Write(message.NewRequest(uint32(index), offset32, blockLength).Encode())
if err != nil {
fmt.Println(err)
}
}
// 监听是否下载完成
<-pc.downloadedCh
return nil
}
func (pc *peerConn) handleUnchoke(readyCh chan<- bool) {
pc.peerChoking = false
// 已经处理过Unchoke信号
if pc.readyComplete {
return
}
// 如果客户端对peer感兴趣并且peer没有choked客户端,就可以开始下载了
if pc.amInterested {
readyCh <- true
} else {
readyCh <- false
}
}
func (pc *peerConn) handleBitfield(buf []byte) {
pc.bitfield = &message.Bitfield{}
pc.bitfield.Decode(buf)
have := pc.getHavePieces(pc.bitfield)
if len(have) > 0 {
// 表示对该peer感兴趣,并且不choked该peer
pc.conn.Write(message.NewInterested().Encode())
pc.amInterested = true
pc.conn.Write(message.NewUnchoke().Encode())
pc.amChoking = false
} else {
pc.conn.Close()
}
}
// 处理下载响应,每次接收到响应直接将block写入到对应文件中
func (pc *peerConn) handlePiece(buf []byte) {
piece := &message.Piece{}
piece.Decode(buf)
info := pc.torrent.MetaInfo.Info
fds := pc.torrent.MetaInfo.GetFileDetails()
blockLength := int64(len(piece.Block))
pieceBegin := int64(piece.Index) * int64(info.PieceLength)
blockBegin := pieceBegin + int64(piece.Begin)
// 计算block对应要写到的文件偏移
var fileBlocks []fileBlock
if len(info.Files) == 0 {
// 单文件
fileBlocks = append(fileBlocks, fileBlock{
filepath: info.Name,
fileSeek: blockBegin,
blockBegin: 0,
blockEnd: blockLength,
})
} else {
// 获取要写入的第一个文件和偏移
writeIndex := getWriteFile(blockBegin, fds)
// block对应种子的偏移
blockFileBegin := blockBegin
// block对应文件的偏移
var blockSeek int64 = 0
for _, f := range fds[writeIndex:] {
// 计算文件可剩余写入字节数
fileWritable := f.End - blockFileBegin
// 计算block剩余写入字节数
blockWritable := blockLength - blockSeek
fb := fileBlock{
filepath: filepath.Join(f.Path...),
fileSeek: blockFileBegin - f.Begin,
blockBegin: blockSeek,
}
if fileWritable >= blockWritable {
// 若够block写入直接跳出循环
fb.blockEnd = blockSeek + blockWritable
fileBlocks = append(fileBlocks, fb)
break
} else {
// 否则计算剩余可写字节数,写入到下一个文件中
fb.blockEnd = blockSeek + fileWritable
fileBlocks = append(fileBlocks, fb)
blockFileBegin = f.End
blockSeek += fileWritable
}
}
}
// 开始写入文件
for _, f := range fileBlocks {
func() {
name := filepath.Join(pc.torrent.Path, f.filepath)
file, err := os.OpenFile(name, os.O_RDWR|os.O_CREATE, 0644)
if err != nil {
panic(err)
}
defer file.Close()
_, err = file.WriteAt(piece.Block[f.blockBegin:f.blockEnd], f.fileSeek)
if err != nil {
panic(err)
}
}()
}
// 出队
<-pc.blockQueue
// piece全部下载完
if atomic.AddInt32(&pc.blockDownloadedCount, 1) == int32(pc.blockCount) {
// 计算piece对应的文件偏移
sha1 := sha1.New()
pieceLength := pc.torrent.MetaInfo.GetPieceLength(int(piece.Index))
writeIndex := getWriteFile(pieceBegin, fds)
fileBegin := pieceBegin - fds[writeIndex].Begin
for _, fd := range fds[writeIndex:] {
func() {
file, err := os.Open(path.Join(pc.torrent.Path, fd.Path[0]))
if err != nil {
panic(err)
}
defer file.Close()
_, err = file.Seek(fileBegin, 0)
if err != nil {
panic(err)
}
written, err := io.CopyN(sha1, file, int64(pieceLength))
if err != nil {
if err != io.EOF {
panic(err)
}
}
pieceLength -= int(written)
}()
if pieceLength > 0 {
// 继续读下个文件
fileBegin = 0
} else {
break
}
}
// 校验piece SHA-1 hash
downHash := [20]byte{}
copy(downHash[:], sha1.Sum(nil))
if downHash == pc.torrent.MetaInfo.Info.Pieces[piece.Index] {
fmt.Printf("piece %d 校验通过\n", piece.Index)
} else {
fmt.Printf("piece %d 校验失败\n", piece.Index)
}
// piece下载完成
pc.downloadedCh <- true
}
}
// 获取piece对应要写入的文件
func getWriteFile(pieceBegin int64, fds []metainfo.FileDetail) int {
for i, f := range fds {
if f.Begin <= pieceBegin && f.End > pieceBegin {
return i
}
}
return -1
}
type fileBlock struct {
filepath string
fileSeek int64
blockBegin int64
blockEnd int64
}
// 获取peer能提供需要下载的文件分片
func (pc *peerConn) getHavePieces(bitfield *message.Bitfield) []int {
states := make([]bool, pc.torrent.PiecesState.Size())
for i := range states {
states[i] = pc.torrent.PiecesState.getState(i) == stateFinished
}
return bitfield.Have(states)
}
// 获取要写入到的文件
/*func (ps *peerConn) getWriteFile(request *message.Request) string {
info := ps.torrent.MetaInfo.Info
// 单文件
if len(info.Files) == 0 {
return info.Name
} else {
request.Index * info.PieceLength + be
for i := 0; i < len(info.Files); i++ {
}
}
}
*/
+166
View File
@@ -0,0 +1,166 @@
package torrent
import (
"fmt"
"github.com/monkeyWie/gopeed/protocol/bt/peer"
"github.com/monkeyWie/gopeed/protocol/bt/tracker"
"os"
"sync"
"testing"
"time"
)
func Test_peerState_ready(t *testing.T) {
ps := getUsablePeer()
fmt.Println("get a peer")
// 下载前准备
err := ps.ready()
fmt.Println("ready end")
if err != nil {
fmt.Println(ps.peer.Address())
t.Fatal(err)
}
}
func Test_peerState_download(t *testing.T) {
psCh := getUsablePeerMore()
lock := sync.Mutex{}
state := map[int]bool{}
for {
select {
case ps := <-psCh:
go func() {
// 下载前准备
err := ps.ready()
if err != nil {
fmt.Println("ready error")
return
}
have := ps.getHavePieces(ps.bitfield)
fmt.Printf("download is ready,have %d\n", len(have))
if len(have) > 0 {
// 获取分片的长度
for _, index := range have {
lock.Lock()
if state[index] {
lock.Unlock()
continue
} else {
state[index] = true
lock.Unlock()
}
fmt.Printf("download index %d\n", index)
ps.downloadPiece(index)
}
return
}
}()
}
}
}
func Test_peerState_downloadPiece(t *testing.T) {
ps := getUsablePeer()
fmt.Println("get a peer")
// 下载前准备
err := ps.ready()
if err != nil {
t.Fatal(err)
}
have := ps.getHavePieces(ps.bitfield)
fmt.Printf("download is ready,have %d\n", len(have))
if len(have) > 0 {
// 获取分片的长度
for _, index := range have {
fmt.Printf("download index %d\n", index)
ps.downloadPiece(index)
}
return
}
}
// 获取一个能连接上的peer
func getUsablePeer() *peerConn {
ch := getUsablePeerMore()
ps := <-ch
return ps
}
// 获取多个能连接上的peer
func getUsablePeerMore() chan *peerConn {
torrent := buildTorrent()
tracker := &tracker.Tracker{
PeerID: torrent.PeerID,
MetaInfo: torrent.MetaInfo,
}
peers, err := tracker.Tracker()
if err != nil {
panic(err)
}
ch := make(chan *peerConn)
for i := range peers {
go peerTest(torrent, &peers[i], ch)
}
return ch
}
func peerTest(torrent *Torrent, peer *peer.Peer, ch chan *peerConn) {
ps := &peerConn{
torrent: torrent,
peer: peer,
}
// 连接至peer
err := ps.ready()
if err != nil {
return
}
ps.conn.Close()
ch <- ps
}
func TestSome(t *testing.T) {
ch := make(chan interface{})
go func() {
time.Sleep(time.Second)
select {
case ch <- nil:
fmt.Println("write")
case <-time.After(time.Second * 3):
fmt.Println("timeout")
}
}()
time.Sleep(time.Second * 5)
r := <-ch
fmt.Println(r)
fmt.Println("111")
}
func TestSome2(t *testing.T) {
wg := sync.WaitGroup{}
wg.Add(10)
file, err := os.OpenFile("e:/testbt/test.data", os.O_RDWR|os.O_CREATE, 0644)
for i := 0; i < 10; i++ {
go func(i int) {
for j := 0; j < 100; j++ {
if err != nil {
panic(err)
}
_, err = file.WriteAt([]byte{byte(i)}, int64(i*8))
}
wg.Done()
}(i)
}
wg.Wait()
}
+82
View File
@@ -0,0 +1,82 @@
package torrent
import (
"github.com/monkeyWie/gopeed/protocol/bt/peer"
"sync"
)
type peerPool struct {
peers []peerState
lock *sync.Mutex
}
type peerState struct {
peer *peer.Peer
using bool
ready bool
}
func newPeerPool() *peerPool {
return &peerPool{
lock: &sync.Mutex{},
}
}
func (pp *peerPool) put(peers []peer.Peer) {
pp.lock.Lock()
defer pp.lock.Unlock()
for i := range peers {
pp.peers = append(pp.peers, peerState{
peer: &peers[i],
using: false,
})
}
}
func (pp *peerPool) get() *peer.Peer {
pp.lock.Lock()
defer pp.lock.Unlock()
for i, p := range pp.peers {
if !p.using {
pp.peers[i].using = true
return pp.peers[i].peer
}
}
return nil
}
func (pp *peerPool) ready(peer *peer.Peer) {
pp.lock.Lock()
defer pp.lock.Unlock()
i := findPeer(pp, peer)
if i != -1 {
pp.peers[i].ready = true
}
}
func (pp *peerPool) release(peer *peer.Peer) {
pp.lock.Lock()
defer pp.lock.Unlock()
i := findPeer(pp, peer)
if i != -1 {
pp.peers[i].using = false
}
}
func (pp *peerPool) remove(peer *peer.Peer) {
pp.lock.Lock()
defer pp.lock.Unlock()
i := findPeer(pp, peer)
if i != -1 {
pp.peers = append(pp.peers[0:i], pp.peers[i+1:]...)
}
}
func findPeer(pp *peerPool, peer *peer.Peer) int {
for i := range pp.peers {
if pp.peers[i].peer == peer {
return i
}
}
return -1
}
+40
View File
@@ -0,0 +1,40 @@
package torrent
import (
"fmt"
"github.com/monkeyWie/gopeed/protocol/bt/peer"
"testing"
)
func Test_PeerPool(t *testing.T) {
pool := newPeerPool()
pool.put([]peer.Peer{
{
IP: 0,
Port: 0,
},
{
IP: 1,
Port: 0,
},
{
IP: 2,
Port: 0,
},
{
IP: 3,
Port: 0,
},
{
IP: 4,
Port: 0,
},
})
get := pool.get()
pool.release(get)
pool.remove(get)
for i := 0; i < 5; i++ {
fmt.Println(pool.get())
}
fmt.Println(pool.get())
}
+56
View File
@@ -0,0 +1,56 @@
package torrent
import "sync"
type state int
const (
stateReady = iota
stateDownloading
stateFinished
)
// 所有piece的下载状态
type piecesState struct {
rw *sync.RWMutex
states []state
}
func NewPiecesState(size int) *piecesState {
ps := &piecesState{
rw: &sync.RWMutex{},
states: make([]state, size),
}
for i := 0; i < size; i++ {
ps.states[i] = stateReady
}
return ps
}
func (ps *piecesState) getState(index int) state {
ps.rw.RLock()
defer ps.rw.RUnlock()
return ps.states[index]
}
func (ps *piecesState) setState(index int, state state) {
ps.rw.Lock()
defer ps.rw.Unlock()
ps.states[index] = state
}
func (ps *piecesState) getReadyAndDownload() int {
ps.rw.Lock()
defer ps.rw.Unlock()
for i, s := range ps.states {
if s == stateReady {
ps.states[i] = stateDownloading
return i
}
}
return -1
}
func (ps *piecesState) Size() int {
return len(ps.states)
}
+100
View File
@@ -0,0 +1,100 @@
package torrent
import (
"fmt"
"github.com/monkeyWie/gopeed/protocol/bt/metainfo"
"github.com/monkeyWie/gopeed/protocol/bt/tracker"
"sync"
"time"
)
type Torrent struct {
PeerID [20]byte
MetaInfo *metainfo.MetaInfo
PiecesState *piecesState
peerPool *peerPool
Path string
}
func NewTorrent(peerID [20]byte, metaInfo *metainfo.MetaInfo) *Torrent {
torrent := &Torrent{
PeerID: peerID,
MetaInfo: metaInfo,
PiecesState: NewPiecesState(len(metaInfo.Info.Pieces)),
}
return torrent
}
func (t *Torrent) Download(path string) {
t.Path = path
t.peerPool = newPeerPool()
t.fetchPeers()
taskCh := make(chan interface{}, 16)
defer close(taskCh)
for {
index := t.PiecesState.getReadyAndDownload()
if index == -1 {
return
}
taskCh <- nil
go func(index int) {
peer := t.peerPool.get()
if peer == nil {
time.Sleep(time.Second * 10)
t.PiecesState.setState(index, stateReady)
<-taskCh
return
}
pc := NewPeerConn(t, peer)
err := pc.ready()
if err != nil {
t.PiecesState.setState(index, stateReady)
t.peerPool.remove(peer)
} else {
fmt.Printf("peer ready:%s,download %d\n", peer.Address(), index)
err = pc.downloadPiece(index)
if err != nil {
fmt.Println(err)
// 下载失败
t.PiecesState.setState(index, stateReady)
} else {
// 下载完成
t.PiecesState.setState(index, stateFinished)
}
t.peerPool.release(peer)
}
<-taskCh
}(index)
}
}
// 获取可用的peer
func (t *Torrent) fetchPeers() {
tracker := &tracker.Tracker{
PeerID: t.PeerID,
MetaInfo: t.MetaInfo,
}
urls := tracker.MetaInfo.AnnounceList[0]
var wg sync.WaitGroup
ch := make(chan interface{})
wg.Add(len(urls))
for _, url := range urls {
go func(url string) {
defer wg.Done()
peers, err := tracker.DoTracker(url)
if err == nil {
fmt.Printf("Tracker end,url:%s,count:%d\n", url, len(peers))
ch <- nil
t.peerPool.put(peers)
}
}(url)
}
go func() {
wg.Wait()
close(ch)
}()
<-ch
}
+44
View File
@@ -0,0 +1,44 @@
package torrent
import (
"github.com/monkeyWie/gopeed/protocol/bt/metainfo"
"github.com/monkeyWie/gopeed/protocol/bt/peer"
"testing"
)
func TestTorrent_Download(t *testing.T) {
torrent := buildTorrent()
torrent.Download("e:/testbt/download")
}
func buildTorrent() *Torrent {
// metaInfo, err := metainfo.ParseFromFile("../testdata/Game.of.Thrones.S08E05.720p.WEB.H264-MEMENTO.torrent")
metaInfo, err := metainfo.ParseFromFile("../testdata/office.torrent")
if err != nil {
panic(err)
}
metaInfo.AnnounceList[0] = append(metaInfo.AnnounceList[0], []string{
"udp://tracker.coppersurfer.tk:6969/announce",
"udp://tracker.leechers-paradise.org:6969/announce",
"udp://tracker.opentrackr.org:1337/announce",
"udp://p4p.arenabg.com:1337/announce",
"udp://9.rarbg.to:2710/announce",
"udp://9.rarbg.me:2710/announce",
"udp://tracker.internetwarriors.net:1337/announce",
"udp://exodus.desync.com:6969/announce",
"udp://tracker.tiny-vps.com:6969/announce",
"udp://tracker.sbsub.com:2710/announce",
"udp://retracker.lanta-net.ru:2710/announce",
"udp://open.stealth.si:80/announce",
"udp://open.demonii.si:1337/announce",
"udp://tracker.torrent.eu.org:451/announce",
"udp://tracker.moeking.me:6969/announce",
"udp://tracker.cyberia.is:6969/announce",
"udp://denis.stalker.upeer.me:6969/announce",
"udp://ipv4.tracker.harry.lu:80/announce",
"udp://zephir.monocul.us:6969/announce",
"udp://xxxtor.com:2710/announce",
}...)
return NewTorrent(peer.GenPeerID(), metaInfo)
}