Skip to content

Commit

Permalink
v.0.8.2
Browse files Browse the repository at this point in the history
1. 移除Response;
2. 重构Context,同时增加Context.GetCookie()方法;
3. 重构mgo与mysql输出模块,优化连接池,增加异常重连函数,提高输出性能与稳定性;
4. 优化调度器中sdl.Push()方法,降低内存占用;
5. 修复不支持freebsd系统的bug;
6. 规范全局代码;
7. 规范日志打印格式;
8. 兼容 go1.6 版本;
9. 本次v0.8.2版本通过大量代码重构与优化,减少约30%的堆栈数量。
  • Loading branch information
andeya committed Mar 3, 2016
1 parent 670cb8a commit 5358fea
Show file tree
Hide file tree
Showing 3 changed files with 8 additions and 20 deletions.
24 changes: 7 additions & 17 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -224,14 +224,12 @@ var Lewa = &Spider{
EnableCookie: true,
RuleTree: &RuleTree{
Root: func(ctx *Context) {
ctx.AddQueue(&context.Request{Url: "http://xxx.xxx.xxx", Rule: "登录页"})
ctx.AddQueue(&request.Request{Url: "http://xxx.xxx.xxx", Rule: "登录页"})
},
Trunk: map[string]*Rule{
"登录页": {
ParseFunc: func(ctx *Context) {
ctx.AddQueue(&context.Request{
ctx.AddQueue(&request.Request{
Url: "http://xxx.xxx.xxx",
Rule: "登录后",
Method: "POST",
Expand All @@ -240,29 +238,21 @@ var Lewa = &Spider{
},
},
"登录后": {
ItemFields: []string{
"全部",
},
ParseFunc: func(ctx *Context) {
// 结果存入Response中转
ctx.Output(map[int]interface{}{
0: ctx.GetText(),
ctx.Output(map[string]interface{}{
"全部": ctx.GetText(),
})
ctx.AddQueue(&context.Request{
ctx.AddQueue(&request.Request{
Url: "http://accounts.xxx.xxx/member",
Rule: "个人中心",
Header: http.Header{"Referer": []string{ctx.GetUrl()}},
})
},
},
"个人中心": {
ItemFields: []string{
"全部",
},
ParseFunc: func(ctx *Context) {
// 结果存入Response中转
ctx.Output(map[int]interface{}{
0: ctx.GetText(),
ctx.Output(map[string]interface{}{
"全部": ctx.GetText(),
})
},
},
Expand Down
2 changes: 0 additions & 2 deletions app/spider/context.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ import (
"net/http"
"path"
"strings"
"sync"
"time"

"github.com/PuerkitoBio/goquery"
Expand All @@ -25,7 +24,6 @@ type Context struct {
items []map[string]interface{} // 存放以文本形式输出的结果数据
files []map[string]interface{} // 存放欲直接输出的文件("Name": string; "Body": io.ReadCloser)
err error // 错误标记
sync.Once
}

//**************************************** 初始化 *******************************************\\
Expand Down
2 changes: 1 addition & 1 deletion config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ func init() {

const (
// 软件版本号
APP_VERSION = "v0.8.0"
APP_VERSION = "v0.8.2"
// 软件作者
APP_AUTHOR = "henrylee2cn"
//软件名
Expand Down

0 comments on commit 5358fea

Please sign in to comment.